diff --git a/idd/LGIdd/CFrameScheduler.cpp b/idd/LGIdd/CFrameScheduler.cpp index 533cdf4c..100e826b 100644 --- a/idd/LGIdd/CFrameScheduler.cpp +++ b/idd/LGIdd/CFrameScheduler.cpp @@ -77,21 +77,25 @@ CFrameScheduler::Client * CFrameScheduler::FindClient(uint32_t clientID) bool CFrameScheduler::ElectOwner(uint64_t now) { - Client * fastest = nullptr; - Client * incumbent = FindClient(m_schedule.clientID); - unsigned subscribers = 0; + Client * fastest = nullptr; + Client * incumbent = FindClient(m_schedule.clientID); + unsigned subscribers = 0; + bool clientExpired = false; for (Client& client : m_clients) { - if (!client.subscribed) + if (client.active && client.expiry <= now) + { + client.active = false; + clientExpired = true; + } + + if (!client.subscribed || !client.ownerCapable) continue; ++subscribers; - if (!client.active || client.expiry <= now) - { - client.active = false; + if (!client.active) continue; - } if (!fastest || client.period < fastest->period) fastest = &client; @@ -100,7 +104,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) if (!subscribers) fastest = nullptr; - if (fastest && incumbent && incumbent->subscribed && incumbent->active && + if (fastest && incumbent && incumbent->subscribed && + incumbent->ownerCapable && incumbent->active && incumbent->expiry > now && incumbent->period <= fastest->period + fastest->period / 200) fastest = incumbent; @@ -110,6 +115,10 @@ bool CFrameScheduler::ElectOwner(uint64_t now) const uint32_t oldEpoch = m_schedule.epoch; const uint64_t oldPeriod = m_schedule.period; const uint64_t oldSlack = m_schedule.targetSlack; + if (incumbent && incumbent->active && + incumbent->generation == oldGeneration) + incumbent->nextDelivery = m_nextDeadline; + if (!fastest) { m_schedule = {}; @@ -134,7 +143,9 @@ bool CFrameScheduler::ElectOwner(uint64_t now) ++m_epoch; m_schedule.epoch = m_epoch; } - m_nextDeadline = m_scheduling ? now + m_schedule.period : 0; + m_nextDeadline = m_scheduling ? fastest->nextDelivery : 0; + if (m_scheduling && !m_nextDeadline) + m_nextDeadline = now + m_schedule.period; m_forceNext = m_scheduling; m_republishNext = m_scheduling; @@ -157,7 +168,8 @@ bool CFrameScheduler::ElectOwner(uint64_t now) } return ownerChanged || oldGeneration != m_schedule.generation || - oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack; + oldPeriod != m_schedule.period || oldSlack != m_schedule.targetSlack || + clientExpired; } void CFrameScheduler::Reset() @@ -191,12 +203,29 @@ void CFrameScheduler::Reset() } void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, - unsigned count, uint64_t now) + unsigned count, const uint32_t * ownerClientIDs, unsigned ownerCount, + uint64_t now) { AcquireSRWLockExclusive(&m_lock); + uint32_t oldClientIDs [LGMP_MAX_CLIENTS] = {}; + bool wasSubscribed [LGMP_MAX_CLIENTS] = {}; + bool wasOwnerCapable[LGMP_MAX_CLIENTS] = {}; + unsigned clientIndex = 0; + for (const Client& client : m_clients) + { + oldClientIDs[clientIndex] = client.clientID; + wasSubscribed[clientIndex] = client.subscribed; + wasOwnerCapable[clientIndex] = client.ownerCapable; + ++clientIndex; + } + + bool subscribersChanged = false; for (Client& client : m_clients) - client.subscribed = false; + { + client.subscribed = false; + client.ownerCapable = false; + } for (unsigned i = 0; i < count; ++i) { @@ -217,12 +246,31 @@ void CFrameScheduler::UpdateSubscribers(const uint32_t * clientIDs, } } + for (unsigned i = 0; i < ownerCount; ++i) + { + Client * client = FindClient(ownerClientIDs[i]); + if (client && client->subscribed) + client->ownerCapable = true; + } + for (Client& client : m_clients) if (client.clientID && !client.subscribed && (client.subscriptionSeen || !client.active || client.expiry <= now)) + { client = {}; + } - const bool changed = ElectOwner(now); + clientIndex = 0; + for (const Client& client : m_clients) + { + if (oldClientIDs[clientIndex] != client.clientID || + wasSubscribed[clientIndex] != client.subscribed || + wasOwnerCapable[clientIndex] != client.ownerCapable) + subscribersChanged = true; + ++clientIndex; + } + + const bool changed = ElectOwner(now) || subscribersChanged; ReleaseSRWLockExclusive(&m_lock); if (changed) WakePublisher(); @@ -248,10 +296,12 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, bool wake = false; if (client) { - client->active = false; - client->expiry = 0; - client->immediate = false; - wake = ElectOwner(now); + client->active = false; + client->expiry = 0; + client->immediate = false; + client->nextDelivery = 0; + wake = true; + wake |= ElectOwner(now); } ReleaseSRWLockExclusive(&m_lock); if (wake) @@ -285,12 +335,16 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, return false; } - bool wake = false; - - if (client->generation != schedule.generation) + const bool reset = client->generation != schedule.generation || + (schedule.flags & KVMFR_FRAME_SCHEDULE_RESET); + bool wake = reset || !client->active || + client->period != schedule.period || + client->targetSlack != schedule.targetSlack; + if (reset) { client->lastFeedbackFrameSerial = 0; client->immediate = false; + client->nextDelivery = now + schedule.period; } client->generation = schedule.generation; client->period = schedule.period; @@ -298,7 +352,10 @@ bool CFrameScheduler::UpdateSchedule(uint32_t sourceClientID, client->expiry = now + static_cast(schedule.lease) * 1000000; client->active = true; if (schedule.flags & KVMFR_FRAME_SCHEDULE_IMMEDIATE) + { client->immediate = true; + wake = true; + } wake |= ElectOwner(now); if (m_scheduling && client->clientID == m_schedule.clientID && client->generation == m_schedule.generation && client->immediate) @@ -346,6 +403,7 @@ bool CFrameScheduler::ApplyFeedback(Client& client, } m_lastPhaseError = schedule.phaseError; client.lastFeedbackFrameSerial = schedule.feedbackFrameSerial; + client.nextDelivery = m_nextDeadline; return correction != 0; } @@ -359,6 +417,23 @@ void CFrameScheduler::AdvanceDeadline(uint64_t now) m_nextDeadline += periods * m_schedule.period; } +void CFrameScheduler::AdvanceDelivery(Client& client, uint64_t now) +{ + if (!client.nextDelivery) + { + client.nextDelivery = now + client.period; + return; + } + + client.nextDelivery += client.period; + if (client.nextDelivery <= now) + { + const uint64_t periods = + (now - client.nextDelivery) / client.period + 1; + client.nextDelivery += periods * client.period; + } +} + bool CFrameScheduler::GetSchedule(Schedule& schedule) const { AcquireSRWLockShared(&m_lock); @@ -463,7 +538,11 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, Client * client = FindClient(m_schedule.clientID); if (client) - client->immediate = false; + { + client->immediate = false; + client->lastDeliveredFrameSerial = frameSerial; + client->deliveredFrameValid = true; + } m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; if (periodic) @@ -471,6 +550,8 @@ void CFrameScheduler::FramePublished(const Schedule& schedule, m_nextDeadline += m_schedule.period; AdvanceDeadline(now); } + if (client) + client->nextDelivery = m_nextDeadline; } ReleaseSRWLockExclusive(&m_lock); } @@ -487,22 +568,134 @@ void CFrameScheduler::FrameRepublished(const Schedule& schedule, Client * client = FindClient(m_schedule.clientID); if (client) - client->immediate = false; + { + client->immediate = false; + client->lastDeliveredFrameSerial = frameSerial; + client->deliveredFrameValid = true; + client->nextDelivery = m_nextDeadline; + } m_lastPublishedFrameSerial = frameSerial; ++m_publishedFrames; } ReleaseSRWLockExclusive(&m_lock); } +unsigned CFrameScheduler::GetSecondaryRecipients( + const uint32_t * clientIDs, unsigned count, uint32_t frameSerial, + uint64_t now, uint32_t * recipients) const +{ + AcquireSRWLockShared(&m_lock); + unsigned recipientCount = 0; + for (unsigned i = 0; i < count; ++i) + { + if (m_scheduling && clientIDs[i] == m_schedule.clientID) + continue; + + const Client * selected = nullptr; + for (const Client& client : m_clients) + if (client.clientID == clientIDs[i]) + { + selected = &client; + break; + } + + const bool active = selected && selected->active && + selected->expiry > now; + const bool sameFrame = selected && selected->deliveredFrameValid && + selected->lastDeliveredFrameSerial == frameSerial; + bool due = !sameFrame; + if (active) + { + const uint64_t target = selected->nextDelivery > + selected->targetSlack ? + selected->nextDelivery - selected->targetSlack : 0; + due = selected->immediate || + (!sameFrame && (!selected->nextDelivery || target <= now)); + } + + if (due) + recipients[recipientCount++] = clientIDs[i]; + } + ReleaseSRWLockShared(&m_lock); + return recipientCount; +} + +bool CFrameScheduler::GetSecondaryTarget(uint32_t frameSerial, + uint64_t now, const uint32_t * blockedClientIDs, + unsigned blockedCount, uint64_t& target) const +{ + bool found = false; + target = now; + + AcquireSRWLockShared(&m_lock); + for (const Client& client : m_clients) + { + if (!client.clientID || !client.subscribed || + (m_scheduling && client.clientID == m_schedule.clientID) || + (client.deliveredFrameValid && + client.lastDeliveredFrameSerial == frameSerial && + !client.immediate)) + continue; + + bool blocked = false; + for (unsigned i = 0; i < blockedCount; ++i) + if (blockedClientIDs[i] == client.clientID) + { + blocked = true; + break; + } + if (blocked) + continue; + + uint64_t candidate = now; + if (client.active && client.expiry > now && !client.immediate && + client.nextDelivery > client.targetSlack) + candidate = client.nextDelivery - client.targetSlack; + if (candidate < now) + candidate = now; + + if (!found || candidate < target) + { + target = candidate; + found = true; + } + } + ReleaseSRWLockShared(&m_lock); + return found; +} + void CFrameScheduler::FrameDelivered(const uint32_t * clientIDs, - unsigned count) + unsigned count, uint32_t frameSerial, uint64_t now) { AcquireSRWLockExclusive(&m_lock); for (unsigned i = 0; i < count; ++i) { Client * client = FindClient(clientIDs[i]); - if (client) - client->immediate = false; + if (!client) + continue; + + const bool immediate = client->immediate; + client->immediate = false; + client->lastDeliveredFrameSerial = frameSerial; + client->deliveredFrameValid = true; + + if (!client->active || client->expiry <= now) + { + client->nextDelivery = 0; + continue; + } + + if (m_scheduling && client->clientID == m_schedule.clientID) + { + client->nextDelivery = m_nextDeadline; + continue; + } + + const bool periodic = !client->nextDelivery || + client->nextDelivery <= now || + client->nextDelivery - now <= client->targetSlack; + if (!immediate || periodic) + AdvanceDelivery(*client, now); } ReleaseSRWLockExclusive(&m_lock); } diff --git a/idd/LGIdd/CFrameScheduler.h b/idd/LGIdd/CFrameScheduler.h index 05d1f38b..c65f39d9 100644 --- a/idd/LGIdd/CFrameScheduler.h +++ b/idd/LGIdd/CFrameScheduler.h @@ -49,11 +49,15 @@ private: uint64_t period; uint64_t targetSlack; uint64_t expiry; + uint64_t nextDelivery; uint32_t lastFeedbackFrameSerial; + uint32_t lastDeliveredFrameSerial; bool subscribed; + bool ownerCapable; bool subscriptionSeen; bool active; bool immediate; + bool deliveredFrameValid; }; mutable SRWLOCK m_lock = SRWLOCK_INIT; @@ -85,6 +89,7 @@ private: bool ElectOwner(uint64_t now); bool ApplyFeedback(Client& client, const KVMFRFrameSchedule& schedule); void AdvanceDeadline(uint64_t now); + static void AdvanceDelivery(Client& client, uint64_t now); void WakePublisher() const; public: @@ -95,7 +100,7 @@ public: void Reset(); void UpdateSubscribers(const uint32_t * clientIDs, unsigned count, - uint64_t now); + const uint32_t * ownerClientIDs, unsigned ownerCount, uint64_t now); bool UpdateSchedule(uint32_t sourceClientID, const KVMFRFrameSchedule& schedule, uint64_t now); bool GetSchedule(Schedule& schedule) const; @@ -108,7 +113,15 @@ public: void FramePublished(const Schedule& schedule, uint32_t frameSerial, uint64_t now, bool periodic); void FrameRepublished(const Schedule& schedule, uint32_t frameSerial); - void FrameDelivered(const uint32_t * clientIDs, unsigned count); + unsigned GetSecondaryRecipients(const uint32_t * clientIDs, + unsigned count, uint32_t frameSerial, uint64_t now, + uint32_t * recipients) const; + bool GetSecondaryTarget(uint32_t frameSerial, uint64_t now, + const uint32_t * blockedClientIDs, unsigned blockedCount, + uint64_t& target) const; + void FrameDelivered(const uint32_t * clientIDs, unsigned count, + uint32_t frameSerial, uint64_t now); + void NotifyPublisher() const { WakePublisher(); } void RecordFrameTiming(uint64_t duration); void LogStatistics(uint64_t now); }; diff --git a/idd/LGIdd/CIndirectDeviceContext.cpp b/idd/LGIdd/CIndirectDeviceContext.cpp index 9a3731f8..cca6d999 100644 --- a/idd/LGIdd/CIndirectDeviceContext.cpp +++ b/idd/LGIdd/CIndirectDeviceContext.cpp @@ -1183,7 +1183,6 @@ bool CIndirectDeviceContext::SetupLGMP(size_t alignSize) m_maxFrameSize = maxFrameSize; m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; m_framePublishSequence = 0; memset(m_frameLastPublishSequence, 0, sizeof(m_frameLastPublishSequence)); @@ -1238,7 +1237,6 @@ void CIndirectDeviceContext::DeInitLGMP() { m_frameScheduler.Reset(); m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; m_framePublishSequence = 0; memset(m_frameLastPublishSequence, 0, sizeof(m_frameLastPublishSequence)); @@ -1259,7 +1257,6 @@ void CIndirectDeviceContext::DeInitLGMP() AcquireSRWLockExclusive(&m_framePublishLock); m_publishedFrameIndex.store(-1, std::memory_order_release); - m_frameResendPending = false; m_framePublishSequence = 0; memset(m_frameLastPublishSequence, 0, sizeof(m_frameLastPublishSequence)); @@ -1318,10 +1315,18 @@ void CIndirectDeviceContext::LGMPTimer() } const uint64_t now = CFrameScheduler::Nanotime(); - uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; - unsigned clientCount = 0; + uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; + uint32_t ownerClientIDs[LGMP_MAX_CLIENTS] = {}; + unsigned clientCount = 0; + unsigned ownerClientCount = 0; LGMP_STATUS subscriberStatus = lgmpHostGetClientIDs( m_frameQueue, clientIDs, &clientCount); + if (subscriberStatus == LGMP_OK) + { + memcpy(ownerClientIDs, clientIDs, + clientCount * sizeof(*ownerClientIDs)); + ownerClientCount = clientCount; + } for (unsigned queueIndex = 0; subscriberStatus == LGMP_OK && queueIndex < LGMP_Q_FRAME_LEN; ++queueIndex) @@ -1333,15 +1338,15 @@ void CIndirectDeviceContext::LGMPTimer() unsigned commonCount = 0; for (unsigned i = 0; - subscriberStatus == LGMP_OK && i < clientCount; + subscriberStatus == LGMP_OK && i < ownerClientCount; ++i) for (unsigned candidate = 0; candidate < queueClientCount; ++candidate) - if (clientIDs[i] == queueClientIDs[candidate]) + if (ownerClientIDs[i] == queueClientIDs[candidate]) { - clientIDs[commonCount++] = clientIDs[i]; + ownerClientIDs[commonCount++] = ownerClientIDs[i]; break; } - clientCount = commonCount; + ownerClientCount = commonCount; } uint8_t data[LGMP_MSGS_SIZE]; @@ -1376,21 +1381,6 @@ void CIndirectDeviceContext::LGMPTimer() sourceClientID, *frameSchedule, now); if (!valid) DEBUG_WARN("Ignoring invalid KVMFR frame schedule"); - else if ((frameSchedule->flags & - (KVMFR_FRAME_SCHEDULE_ACTIVE | - KVMFR_FRAME_SCHEDULE_IMMEDIATE)) == - (KVMFR_FRAME_SCHEDULE_ACTIVE | - KVMFR_FRAME_SCHEDULE_IMMEDIATE)) - { - CFrameScheduler::Schedule owner = {}; - if (!m_frameScheduler.GetSchedule(owner) || - owner.clientID != sourceClientID) - { - AcquireSRWLockExclusive(&m_framePublishLock); - m_frameResendPending = true; - ReleaseSRWLockExclusive(&m_framePublishLock); - } - } break; } } @@ -1399,7 +1389,9 @@ void CIndirectDeviceContext::LGMPTimer() } if (subscriberStatus == LGMP_OK) - m_frameScheduler.UpdateSubscribers(clientIDs, clientCount, now); + m_frameScheduler.UpdateSubscribers( + clientIDs, clientCount, + ownerClientIDs, ownerClientCount, now); else DEBUG_WARN("Failed to query LGMP frame subscribers: %s", lgmpStatusString(subscriberStatus)); @@ -1407,11 +1399,7 @@ void CIndirectDeviceContext::LGMPTimer() m_frameScheduler.LogStatistics(now); if (lgmpHostQueueNewSubs(m_frameQueue)) - { - AcquireSRWLockExclusive(&m_framePublishLock); - m_frameResendPending = true; - ReleaseSRWLockExclusive(&m_framePublishLock); - } + m_frameScheduler.NotifyPublisher(); ProcessFrameDeliveries(); @@ -1422,12 +1410,13 @@ void CIndirectDeviceContext::LGMPTimer() } } -bool CIndirectDeviceContext::PostSharedFrame(unsigned frameIndex, - uint32_t excludeClientID) +CIndirectDeviceContext::SharedFramePostResult +CIndirectDeviceContext::PostSharedFrame(unsigned frameIndex, + uint32_t excludeClientID, uint64_t now) { if (frameIndex >= LGMP_Q_FRAME_BUFFER_LEN || lgmpHostQueuePending(m_frameQueue) != 0) - return false; + return SHARED_FRAME_FAILED; uint32_t clientIDs[LGMP_MAX_CLIENTS] = {}; unsigned clientCount = 0; @@ -1437,22 +1426,28 @@ bool CIndirectDeviceContext::PostSharedFrame(unsigned frameIndex, { DEBUG_ERROR("Failed to query shared frame subscribers: %s", lgmpStatusString(status)); - return false; + return SHARED_FRAME_FAILED; } + uint32_t recipients[LGMP_MAX_CLIENTS] = {}; + const uint32_t frameSerial = m_frame[frameIndex]->frameSerial; + const unsigned recipientCount = + m_frameScheduler.GetSecondaryRecipients( + clientIDs, clientCount, frameSerial, now, recipients); + unsigned targetCount = 0; - for (unsigned i = 0; i < clientCount; ++i) + for (unsigned i = 0; i < recipientCount; ++i) { - bool excluded = clientIDs[i] == excludeClientID; + bool excluded = recipients[i] == excludeClientID; for (const OwnerDelivery& delivery : m_ownerDelivery) - if (delivery.active && delivery.clientID == clientIDs[i]) + if (delivery.active && delivery.clientID == recipients[i]) { excluded = true; break; } if (!excluded) - clientIDs[targetCount++] = clientIDs[i]; + recipients[targetCount++] = recipients[i]; } if (!targetCount) @@ -1461,42 +1456,38 @@ bool CIndirectDeviceContext::PostSharedFrame(unsigned frameIndex, m_frameDelivery[frameIndex].sharedOwnerClientID = 0; m_frameDelivery[frameIndex].sharedOwnerPending = false; m_frameDelivery[frameIndex].sharedPending = false; - m_frameDelivery[frameIndex].sharedDelivered = true; - m_frameResendPending = false; - return true; + return SHARED_FRAME_IDLE; } - unsigned recipientCount = 0; + unsigned postedCount = 0; status = lgmpHostQueuePostForClients( m_frameQueue, 0, m_frameMemory[frameIndex], - clientIDs, targetCount, &recipientCount); + recipients, targetCount, &postedCount); if (status != LGMP_OK) { if (status != LGMP_ERR_QUEUE_FULL) DEBUG_ERROR("Failed to publish shared frame: %s", lgmpStatusString(status)); - return false; + return SHARED_FRAME_FAILED; } - if (!recipientCount) + if (!postedCount) { m_frameDelivery[frameIndex].sharedOwnerToken = 0; m_frameDelivery[frameIndex].sharedOwnerClientID = 0; m_frameDelivery[frameIndex].sharedOwnerPending = false; m_frameDelivery[frameIndex].sharedPending = false; - m_frameDelivery[frameIndex].sharedDelivered = true; - m_frameResendPending = false; - return true; + return SHARED_FRAME_IDLE; } m_frameDelivery[frameIndex].sharedOwnerToken = 0; m_frameDelivery[frameIndex].sharedOwnerClientID = 0; m_frameDelivery[frameIndex].sharedOwnerPending = false; m_frameDelivery[frameIndex].sharedPending = true; - m_frameDelivery[frameIndex].sharedDelivered = true; - m_frameResendPending = false; - m_frameScheduler.FrameDelivered(clientIDs, targetCount); - return true; + if (postedCount == targetCount) + m_frameScheduler.FrameDelivered( + recipients, targetCount, frameSerial, now); + return SHARED_FRAME_POSTED; } bool CIndirectDeviceContext::PostSharedOwnerFrame(unsigned frameIndex, @@ -1523,8 +1514,6 @@ bool CIndirectDeviceContext::PostSharedOwnerFrame(unsigned frameIndex, delivery.sharedOwnerClientID = schedule.clientID; delivery.sharedOwnerPending = true; delivery.sharedPending = true; - if (!delivery.sharedDelivered) - m_frameResendPending = true; return true; } @@ -1538,28 +1527,26 @@ void CIndirectDeviceContext::ProcessFrameDeliveries() AcquireSRWLockExclusive(&m_framePublishLock); + bool released = false; for (unsigned i = 0; i < LGMP_Q_FRAME_BUFFER_LEN; ++i) { if (m_frameDelivery[i].sharedOwnerPending && !lgmpHostQueueMessagePending( m_frameQueue, m_frameMemory[i], m_frameDelivery[i].sharedOwnerToken)) + { m_frameDelivery[i].sharedOwnerPending = false; + released = true; + } if (m_frameDelivery[i].sharedPending && !lgmpHostQueuePayloadPending(m_frameQueue, m_frameMemory[i])) + { m_frameDelivery[i].sharedPending = false; + released = true; + } } - CFrameScheduler::Schedule schedule = {}; - const bool scheduled = m_frameScheduler.GetSchedule(schedule); - const uint64_t scheduleToken = scheduled ? - FrameScheduleToken(schedule) : 0; - const LONG publishedFrameIndex = - m_publishedFrameIndex.load(std::memory_order_acquire); - - int newestAcked = -1; - uint32_t newestAckedClient = 0; for (unsigned queueIndex = 0; queueIndex < LGMP_Q_FRAME_LEN; ++queueIndex) @@ -1572,91 +1559,15 @@ void CIndirectDeviceContext::ProcessFrameDeliveries() continue; const unsigned frameIndex = owner.frameIndex; - const bool latestDelivery = - static_cast(frameIndex) == publishedFrameIndex && - m_frameLastPublishSequence[frameIndex] == m_framePublishSequence; - const bool authoritative = (!scheduled && latestDelivery) || - (scheduled && owner.clientID == schedule.clientID && - owner.token == scheduleToken); - if (authoritative && - (newestAcked < 0 || - m_frameLastPublishSequence[frameIndex] > - m_frameLastPublishSequence[newestAcked])) - { - newestAcked = static_cast(frameIndex); - newestAckedClient = owner.clientID; - } - else if (!authoritative && !latestDelivery) - m_frameResendPending = true; - m_frameDelivery[frameIndex].ownerQueueMask &= ~(1U << queueIndex); owner = {}; - } - - if (newestAcked >= 0) - { - FrameDelivery& delivery = m_frameDelivery[newestAcked]; - const bool needsShared = !delivery.sharedDelivered; - - const bool latest = newestAcked == publishedFrameIndex && - m_frameLastPublishSequence[newestAcked] == m_framePublishSequence; - if (needsShared && - (!latest || - m_frameInFlight[newestAcked].load(std::memory_order_acquire) || - lgmpHostQueuePending(m_frameQueue) != 0)) - { - m_frameResendPending = true; - } - else if (needsShared && - !PostSharedFrame( - static_cast(newestAcked), newestAckedClient)) - { - m_frameResendPending = true; - } - } - - if (m_frameResendPending && - lgmpHostQueuePending(m_frameQueue) == 0) - { - const LONG frameIndex = - m_publishedFrameIndex.load(std::memory_order_acquire); - if (frameIndex >= 0 && - !m_frameDelivery[frameIndex].sharedPending && - !m_frameInFlight[frameIndex].load(std::memory_order_acquire)) - { - FrameDelivery& delivery = m_frameDelivery[frameIndex]; - bool ownerDelivered = !scheduled; - if (scheduled) - { - ownerDelivered = - delivery.sharedOwnerClientID == schedule.clientID && - delivery.sharedOwnerToken == scheduleToken; - for (const OwnerDelivery& owner : m_ownerDelivery) - if (owner.active && - owner.clientID == schedule.clientID && - owner.token == scheduleToken && - owner.frameIndex == static_cast(frameIndex)) - { - ownerDelivered = true; - break; - } - } - - const bool reserveSharedOwner = scheduled && - delivery.sharedOwnerClientID == schedule.clientID && - delivery.sharedOwnerToken == scheduleToken && - FindAvailableOwnerQueue(static_cast(frameIndex)) < 0; - if (ownerDelivered && !reserveSharedOwner) - { - const uint32_t excludeClientID = scheduled ? - schedule.clientID : delivery.sharedOwnerClientID; - PostSharedFrame( - static_cast(frameIndex), excludeClientID); - } - } + released = true; } ReleaseSRWLockExclusive(&m_framePublishLock); + + if (released) + m_frameScheduler.NotifyPublisher(); } int CIndirectDeviceContext::FindAvailableOwnerQueue( @@ -1781,6 +1692,60 @@ void CIndirectDeviceContext::ProcessFrameQueue() ProcessFrameDeliveries(); } +bool CIndirectDeviceContext::GetSharedFrameTarget(uint64_t now, + uint64_t& target) +{ + if (!m_frameQueue) + return false; + + AcquireSRWLockShared(&m_framePublishLock); + const LONG frameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + if (frameIndex < 0 || + m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + lgmpHostQueuePending(m_frameQueue) != 0) + { + ReleaseSRWLockShared(&m_framePublishLock); + return false; + } + + uint32_t blockedClientIDs[LGMP_Q_FRAME_LEN] = {}; + unsigned blockedCount = 0; + for (const OwnerDelivery& delivery : m_ownerDelivery) + if (delivery.active) + blockedClientIDs[blockedCount++] = delivery.clientID; + + const bool result = m_frameScheduler.GetSecondaryTarget( + m_frame[frameIndex]->frameSerial, now, + blockedClientIDs, blockedCount, target); + ReleaseSRWLockShared(&m_framePublishLock); + return result; +} + +bool CIndirectDeviceContext::ReplaySharedFrame(uint64_t now, bool& retry) +{ + retry = false; + if (!m_frameQueue) + return false; + + AcquireSRWLockExclusive(&m_framePublishLock); + const LONG frameIndex = + m_publishedFrameIndex.load(std::memory_order_acquire); + if (frameIndex < 0 || + m_frameInFlight[frameIndex].load(std::memory_order_acquire) || + lgmpHostQueuePending(m_frameQueue) != 0) + { + ReleaseSRWLockExclusive(&m_framePublishLock); + return false; + } + + const SharedFramePostResult result = PostSharedFrame( + static_cast(frameIndex), 0, now); + ReleaseSRWLockExclusive(&m_framePublishLock); + retry = result == SHARED_FRAME_FAILED; + return result == SHARED_FRAME_POSTED; +} + CIndirectDeviceContext::PreparedFrameBuffer CIndirectDeviceContext::PrepareFrameBuffer( unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects) @@ -1950,6 +1915,7 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, if (!m_frameQueue || frameIndex >= LGMP_Q_FRAME_BUFFER_LEN) return false; + const uint64_t now = CFrameScheduler::Nanotime(); AcquireSRWLockExclusive(&m_framePublishLock); CFrameScheduler::Schedule currentSchedule = {}; const bool scheduling = @@ -1991,17 +1957,15 @@ bool CIndirectDeviceContext::PublishFrameBuffer(unsigned frameIndex, FrameDelivery& delivery = m_frameDelivery[frameIndex]; delivery.ownerQueueMask |= 1U << queueIndex; - delivery.sharedDelivered = false; - published = true; + published = true; - if (!PostSharedFrame(frameIndex, schedule.clientID)) - m_frameResendPending = true; + PostSharedFrame(frameIndex, schedule.clientID, now); } } } } else - published = PostSharedFrame(frameIndex, 0); + published = PostSharedFrame(frameIndex, 0, now) != SHARED_FRAME_FAILED; if (published) { diff --git a/idd/LGIdd/CIndirectDeviceContext.h b/idd/LGIdd/CIndirectDeviceContext.h index a56d14e9..87019877 100644 --- a/idd/LGIdd/CIndirectDeviceContext.h +++ b/idd/LGIdd/CIndirectDeviceContext.h @@ -111,10 +111,16 @@ private: std::atomic m_publishedFrameIndex = -1; std::atomic m_frameInFlight[LGMP_Q_FRAME_BUFFER_LEN] = {}; SRWLOCK m_framePublishLock = SRWLOCK_INIT; - bool m_frameResendPending = false; uint64_t m_framePublishSequence = 0; uint64_t m_frameLastPublishSequence[LGMP_Q_FRAME_BUFFER_LEN] = {}; + enum SharedFramePostResult + { + SHARED_FRAME_FAILED, + SHARED_FRAME_IDLE, + SHARED_FRAME_POSTED, + }; + struct FrameDelivery { uint64_t sharedOwnerToken = 0; @@ -122,7 +128,6 @@ private: uint32_t sharedOwnerClientID = 0; bool sharedOwnerPending = false; bool sharedPending = false; - bool sharedDelivered = false; }; struct OwnerDelivery @@ -185,7 +190,8 @@ private: int FindOwnerDelivery(uint32_t clientID) const; int FindSharedOwnerDelivery(uint32_t clientID) const; bool HasOwnerDelivery(uint32_t clientID) const; - bool PostSharedFrame(unsigned frameIndex, uint32_t excludeClientID); + SharedFramePostResult PostSharedFrame(unsigned frameIndex, + uint32_t excludeClientID, uint64_t now); bool PostSharedOwnerFrame(unsigned frameIndex, const CFrameScheduler::Schedule& schedule); void ResendCursor(); @@ -264,6 +270,8 @@ public: return m_publishedFrameIndex.load(std::memory_order_acquire) >= 0; } void ProcessFrameQueue(); + bool GetSharedFrameTarget(uint64_t now, uint64_t& target); + bool ReplaySharedFrame(uint64_t now, bool& retry); PreparedFrameBuffer PrepareFrameBuffer(unsigned pitch, const D12FrameFormat& srcFormat, const D12FrameFormat& dstFormat, const RECT * dirtyRects, unsigned nbDirtyRects); bool PublishFrameBuffer(unsigned frameIndex, const CFrameScheduler::Schedule& schedule); diff --git a/idd/LGIdd/CSwapChainProcessor.cpp b/idd/LGIdd/CSwapChainProcessor.cpp index 67dd66a5..a394228c 100644 --- a/idd/LGIdd/CSwapChainProcessor.cpp +++ b/idd/LGIdd/CSwapChainProcessor.cpp @@ -231,11 +231,15 @@ void CSwapChainProcessor::PublisherThread() m_devContext->GetPublishTarget( now, target, schedule, periodic, republish); - if (!HasReadyCandidate()) + const bool ready = HasReadyCandidate(); + if (!ready) { + m_devContext->ProcessFrameQueue(); + if (HasReadyCandidate()) + continue; + if (republish && m_devContext->HasPublishedFrame()) { - m_devContext->ProcessFrameQueue(); if (m_devContext->RepublishFrameBuffer(schedule)) continue; @@ -247,6 +251,37 @@ void CSwapChainProcessor::PublisherThread() continue; } + const uint64_t replayNow = CFrameScheduler::Nanotime(); + uint64_t replayTarget; + if (m_devContext->GetSharedFrameTarget(replayNow, replayTarget)) + { + bool retry = false; + if (replayTarget <= replayNow) + { + if (m_devContext->ReplaySharedFrame(replayNow, retry)) + continue; + if (!retry) + { + if (m_publishTimer.Get()) + CancelWaitableTimer(m_publishTimer.Get()); + if (WaitForMultipleObjects( + ARRAYSIZE(idleHandles), idleHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + } + + const uint64_t delay = replayTarget > replayNow ? + replayTarget - replayNow : PUBLISH_RETRY_NS; + ArmPublishTimer(m_publishTimer.Get(), delay); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + if (m_publishTimer.Get()) CancelWaitableTimer(m_publishTimer.Get()); if (WaitForMultipleObjects( @@ -256,6 +291,34 @@ void CSwapChainProcessor::PublisherThread() continue; } + uint64_t replayTarget; + if (m_devContext->GetSharedFrameTarget(now, replayTarget) && + replayTarget < target) + { + if (replayTarget <= now) + { + m_devContext->ProcessFrameQueue(); + bool retry = false; + if (m_devContext->ReplaySharedFrame( + CFrameScheduler::Nanotime(), retry)) + continue; + + if (retry) + replayTarget = now + PUBLISH_RETRY_NS; + else + replayTarget = target; + } + + const uint64_t delay = replayTarget > now ? + replayTarget - now : PUBLISH_RETRY_NS; + ArmPublishTimer(m_publishTimer.Get(), delay); + if (WaitForMultipleObjects( + ARRAYSIZE(timerHandles), timerHandles, FALSE, INFINITE) == + WAIT_OBJECT_0) + break; + continue; + } + if (target > now) { ArmPublishTimer(m_publishTimer.Get(), target - now);