diff --git a/idd/LGAudio/Adapter.cpp b/idd/LGAudio/Adapter.cpp index 225cef1c..aec3348c 100644 --- a/idd/LGAudio/Adapter.cpp +++ b/idd/LGAudio/Adapter.cpp @@ -112,33 +112,140 @@ NTSTATUS InstallWaveSubdevice( } CAudioAdapter::CAudioAdapter() : - m_refCount(1) + m_refCount(1), + m_brokerActive(0), + m_renderDiscontinuity(0), + m_renderState{}, + m_captureState{} { + KeInitializeSpinLock(&m_transportLock); + KeInitializeSpinLock(&m_streamStateLock); + m_renderState.state = KSSTATE_STOP; + m_captureState.state = KSSTATE_STOP; } CAudioAdapter::~CAudioAdapter() { - m_ring.Destroy(); + m_captureRing.Destroy(); + m_renderRing.Destroy(); } NTSTATUS CAudioAdapter::Initialize() { NTSTATUS status = m_clock.Initialize(); if (NT_SUCCESS(status)) - status = m_ring.Init(); + status = m_renderRing.Init(); + if (NT_SUCCESS(status)) + status = m_captureRing.Init(); return status; } + _IRQL_requires_max_(DISPATCH_LEVEL) -NTSTATUS CAudioAdapter::ConfigureRender( +NTSTATUS CAudioAdapter::ConfigureStream( + _In_ EndpointKind kind, _In_ const FormatFingerprint &format) { - return m_ring.Configure(format); + CAudioRingBuffer &ring = kind == EndpointKind::Render ? + m_renderRing : m_captureRing; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + const NTSTATUS status = ring.Configure(format); + if (!NT_SUCCESS(status)) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return status; + } + + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + AudioStreamSnapshot &stream = kind == EndpointKind::Render ? + m_renderState : m_captureState; + stream.format = format; + stream.formatValid = true; + ++stream.generation; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); + KeReleaseSpinLock(&m_transportLock, transportIrql); + + BrokerDeviceNotifyStreamChanged(this, kind); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::SetStreamState( + _In_ EndpointKind kind, + _In_ KSSTATE state) +{ + bool changed; + bool reset; + + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + AudioStreamSnapshot &stream = kind == EndpointKind::Render ? + m_renderState : m_captureState; + changed = stream.state != state; + reset = changed && state == KSSTATE_STOP; + stream.state = state; + if (reset) + ++stream.generation; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); + + if (reset) + { + if (kind == EndpointKind::Render) + m_renderRing.Reset(); + else + m_captureRing.Reset(); + } + KeReleaseSpinLock(&m_transportLock, transportIrql); + if (changed) + BrokerDeviceNotifyStreamChanged(this, kind); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::GetStreamSnapshot( + _In_ EndpointKind kind, + _Out_ AudioStreamSnapshot *snapshot) +{ + if (!snapshot) + return; + + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + *snapshot = kind == EndpointKind::Render ? + m_renderState : m_captureState; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::SetBrokerActive(_In_ bool active) +{ + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const LONG value = active ? 1 : 0; + if (InterlockedExchange(&m_brokerActive, value) == value) + { + KeReleaseSpinLock(&m_transportLock, oldIrql); + return; + } + + m_captureRing.Reset(); + m_renderRing.Reset(); + InterlockedExchange(&m_renderDiscontinuity, active ? 1 : 0); + KeReleaseSpinLock(&m_transportLock, oldIrql); } _IRQL_requires_max_(DISPATCH_LEVEL) void CAudioAdapter::ResetCapture() { - m_ring.Reset(); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + if (InterlockedCompareExchange(&m_brokerActive, 0, 0)) + m_captureRing.Reset(); + else + m_renderRing.Reset(); + KeReleaseSpinLock(&m_transportLock, oldIrql); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -146,7 +253,14 @@ void CAudioAdapter::WriteRenderFrames( _In_reads_bytes_(bytes) const BYTE *data, _In_ ULONG bytes) { - m_ring.Push(data, bytes); + ULONG droppedBytes = 0; + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + m_renderRing.Push(data, bytes, &droppedBytes); + if (droppedBytes) + InterlockedExchange(&m_renderDiscontinuity, 1); + KeReleaseSpinLock(&m_transportLock, oldIrql); + BrokerDeviceNotifyRenderAvailable(this, bytes, droppedBytes); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -155,7 +269,16 @@ void CAudioAdapter::ReadCaptureFrames( _Out_writes_bytes_(bytes) BYTE *data, _In_ ULONG bytes) { - m_ring.Pop(format, data, bytes); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const bool brokerActive = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0; + CAudioRingBuffer &ring = brokerActive ? + m_captureRing : m_renderRing; + const ULONG copied = ring.Pop(format, data, bytes); + KeReleaseSpinLock(&m_transportLock, oldIrql); + if (brokerActive && copied < bytes) + BrokerDeviceRecordCaptureUnderrun(this, bytes - copied); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -163,7 +286,107 @@ void CAudioAdapter::DiscardCaptureFrames( _In_ const FormatFingerprint &format, _In_ ULONGLONG bytes) { - m_ring.Discard(format, bytes); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const bool brokerActive = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0; + CAudioRingBuffer &ring = brokerActive ? + m_captureRing : m_renderRing; + const ULONG discarded = ring.Discard(format, bytes); + KeReleaseSpinLock(&m_transportLock, oldIrql); + if (brokerActive && bytes > discarded) + BrokerDeviceRecordCaptureUnderrun(this, bytes - discarded); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS CAudioAdapter::ReadRenderFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _Out_writes_bytes_(capacity) BYTE *data, + _In_ ULONG capacity, + _Out_ PULONG bytesRead, + _Out_ bool *discontinuity) +{ + if (!data || !bytesRead || !discontinuity || !format.blockAlign) + return STATUS_INVALID_PARAMETER; + + *bytesRead = 0; + *discontinuity = false; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + + KIRQL stateIrql; + KeAcquireSpinLock(&m_streamStateLock, &stateIrql); + const bool active = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0 && + m_renderState.state == KSSTATE_RUN && + m_renderState.formatValid && + m_renderState.generation == streamGeneration && + EqualFormatFingerprint(m_renderState.format, format); + KeReleaseSpinLock(&m_streamStateLock, stateIrql); + if (!active) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_DEVICE_NOT_READY; + } + + ULONG bytes = m_renderRing.Available(); + if (bytes > capacity) + bytes = capacity; + bytes -= bytes % format.blockAlign; + if (!bytes) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_SUCCESS; + } + + const ULONG copied = m_renderRing.Pop(format, data, bytes); + if (copied) + { + *discontinuity = InterlockedExchange( + &m_renderDiscontinuity, 0) != 0; + } + *bytesRead = copied; + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS CAudioAdapter::WriteCaptureFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _In_ bool discontinuity, + _In_reads_bytes_(bytes) const BYTE *data, + _In_ ULONG bytes, + _Out_ PULONG droppedBytes) +{ + if (!data || !droppedBytes) + return STATUS_INVALID_PARAMETER; + + *droppedBytes = 0; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + + KIRQL stateIrql; + KeAcquireSpinLock(&m_streamStateLock, &stateIrql); + const bool active = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0 && + m_captureState.state == KSSTATE_RUN && + m_captureState.formatValid && + m_captureState.generation == streamGeneration && + EqualFormatFingerprint(m_captureState.format, format); + KeReleaseSpinLock(&m_streamStateLock, stateIrql); + if (!active) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_DEVICE_NOT_READY; + } + + if (discontinuity) + m_captureRing.Reset(); + const ULONG written = m_captureRing.Push(data, bytes, droppedBytes); + KeReleaseSpinLock(&m_transportLock, transportIrql); + return written == bytes ? STATUS_SUCCESS : STATUS_INVALID_PARAMETER; } _IRQL_requires_max_(DISPATCH_LEVEL) diff --git a/idd/LGAudio/Adapter.h b/idd/LGAudio/Adapter.h index 7174ff34..bca4820a 100644 --- a/idd/LGAudio/Adapter.h +++ b/idd/LGAudio/Adapter.h @@ -24,6 +24,15 @@ #include "Common.h" #include "RingBuffer.h" +struct AudioStreamSnapshot +{ + KSSTATE state; + bool formatValid; + UCHAR reserved[3]; + ULONGLONG generation; + FormatFingerprint format; +}; + class CAudioAdapter final : public IUnknown { public: @@ -33,9 +42,23 @@ public: NTSTATUS Initialize(); _IRQL_requires_max_(DISPATCH_LEVEL) - NTSTATUS ConfigureRender( + NTSTATUS ConfigureStream( + _In_ EndpointKind kind, _In_ const FormatFingerprint &format); + _IRQL_requires_max_(DISPATCH_LEVEL) + void SetStreamState( + _In_ EndpointKind kind, + _In_ KSSTATE state); + + _IRQL_requires_max_(DISPATCH_LEVEL) + void GetStreamSnapshot( + _In_ EndpointKind kind, + _Out_ AudioStreamSnapshot *snapshot); + + _IRQL_requires_max_(DISPATCH_LEVEL) + void SetBrokerActive(_In_ bool active); + _IRQL_requires_max_(DISPATCH_LEVEL) void ResetCapture(); @@ -55,6 +78,24 @@ public: _In_ const FormatFingerprint &format, _In_ ULONGLONG bytes); + _IRQL_requires_max_(DISPATCH_LEVEL) + NTSTATUS ReadRenderFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _Out_writes_bytes_(capacity) BYTE *data, + _In_ ULONG capacity, + _Out_ PULONG bytesRead, + _Out_ bool *discontinuity); + + _IRQL_requires_max_(DISPATCH_LEVEL) + NTSTATUS WriteCaptureFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _In_ bool discontinuity, + _In_reads_bytes_(bytes) const BYTE *data, + _In_ ULONG bytes, + _Out_ PULONG droppedBytes); + _IRQL_requires_max_(DISPATCH_LEVEL) AudioClockSnapshot GetClockSnapshot() const; @@ -66,8 +107,15 @@ public: private: volatile LONG m_refCount; + volatile LONG m_brokerActive; + volatile LONG m_renderDiscontinuity; + KSPIN_LOCK m_transportLock; + KSPIN_LOCK m_streamStateLock; + AudioStreamSnapshot m_renderState; + AudioStreamSnapshot m_captureState; CAudioClock m_clock; - CAudioRingBuffer m_ring; + CAudioRingBuffer m_renderRing; + CAudioRingBuffer m_captureRing; }; NTSTATUS StartAudioDevice( diff --git a/idd/LGAudio/BrokerDevice.cpp b/idd/LGAudio/BrokerDevice.cpp index 3ef147c3..f660755e 100644 --- a/idd/LGAudio/BrokerDevice.cpp +++ b/idd/LGAudio/BrokerDevice.cpp @@ -31,6 +31,9 @@ extern "C" namespace { constexpr ULONG kBrokerFileSignature = 'fBGL'; + constexpr ULONG kBrokerReadPoolTag = 'qBGL'; + constexpr LONG kMaximumPendingReads = 32; + constexpr ULONG kReadServiceBatch = 8; struct BrokerFileContext { @@ -39,13 +42,33 @@ namespace UCHAR reserved[3]; ULONGLONG adapterGeneration; ULONGLONG bindingGeneration; + ULONGLONG captureStreamGeneration; + ULONGLONG nextCaptureSequence; + CAudioAdapter *adapter; }; struct BrokerControlExtension { KSPIN_LOCK stateLock; + FAST_MUTEX bindingLock; + FAST_MUTEX captureWriteLock; + EX_RUNDOWN_REF bindingRundown; + bool bindingRundownCompleted; PFILE_OBJECT owner; BrokerFileContext file; + IO_CSQ readCsq; + KSPIN_LOCK readQueueLock; + LIST_ENTRY readQueue; + KSPIN_LOCK pendingReadLock; + LONG pendingReadCount; + KEVENT readsDrainedEvent; + }; + + struct BrokerReadContext + { + ULONGLONG adapterGeneration; + ULONGLONG bindingGeneration; + ULONGLONG streamGeneration; }; struct AdapterSnapshot @@ -53,23 +76,36 @@ namespace bool present; ULONGLONG generation; ULONGLONG eventSequence; + AudioStreamSnapshot render; + AudioStreamSnapshot capture; + }; + + struct BindingSnapshot + { + CAudioAdapter *adapter; + ULONGLONG adapterGeneration; + ULONGLONG bindingGeneration; }; FAST_MUTEX g_controlLifecycleLock; FAST_MUTEX g_adapterLifecycleLock; KSPIN_LOCK g_adapterStateLock; EX_RUNDOWN_REF g_adapterRundown; + EX_RUNDOWN_REF g_controlServiceRundown; + KDPC g_readServiceDpc; PDRIVER_OBJECT g_driverObject; PDEVICE_OBJECT g_controlDevice; CAudioAdapter *g_adapter; PDEVICE_OBJECT g_adapterDevice; + CAudioAdapter *g_boundAdapter; volatile LONG g_initialized; volatile LONG g_retiring; volatile LONG g_pnpDeviceCount; bool g_linkCreated; bool g_rundownCompleted; + bool g_controlServiceRundownCompleted; volatile LONG64 g_adapterGeneration; volatile LONG64 g_eventSequence; @@ -78,6 +114,21 @@ namespace volatile LONG64 g_unbindCount; volatile LONG64 g_cleanupCount; volatile LONG64 g_rejectedRequestCount; + volatile LONG64 g_renderPacketCount; + volatile LONG64 g_renderByteCount; + volatile LONG64 g_renderDropBytes; + volatile LONG64 g_capturePacketCount; + volatile LONG64 g_captureByteCount; + volatile LONG64 g_captureDropBytes; + volatile LONG64 g_captureUnderrunBytes; + volatile LONG64 g_cancelledReadCount; + volatile LONG64 g_renderSequence; + + void ReadServiceDpc( + _In_ PKDPC dpc, + _In_opt_ PVOID deferredContext, + _In_opt_ PVOID systemArgument1, + _In_opt_ PVOID systemArgument2); BrokerControlExtension *GetControlExtension( _In_ PDEVICE_OBJECT deviceObject) @@ -97,6 +148,13 @@ namespace InterlockedIncrement64(&g_eventSequence); } + void AddCounter( + _Inout_ volatile LONG64 *counter, + _In_ ULONGLONG value) + { + InterlockedAdd64(counter, static_cast(value)); + } + PDEVICE_OBJECT LoadControlDevice() { return static_cast( @@ -113,6 +171,14 @@ namespace deviceObject)); } + CAudioAdapter *LoadBoundAdapter() + { + return static_cast( + InterlockedCompareExchangePointer( + reinterpret_cast(&g_boundAdapter), + nullptr, nullptr)); + } + NTSTATUS CompleteIrp( _Inout_ PIRP irp, _In_ NTSTATUS status, @@ -124,16 +190,196 @@ namespace return status; } + LONG AddPendingRead(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + const LONG count = ++extension->pendingReadCount; + if (count == 1) + KeClearEvent(&extension->readsDrainedEvent); + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + return count; + } + + void ReleasePendingRead(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + NT_ASSERT(extension->pendingReadCount > 0); + const LONG count = --extension->pendingReadCount; + if (count == 0) + KeSetEvent(&extension->readsDrainedEvent, IO_NO_INCREMENT, FALSE); + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + } + + bool HasPendingReads(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + const bool pending = extension->pendingReadCount != 0; + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + return pending; + } + + BrokerControlExtension *GetCsqExtension(_In_ PIO_CSQ csq) + { + return CONTAINING_RECORD(csq, BrokerControlExtension, readCsq); + } + + void CsqInsertIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + InsertTailList(&extension->readQueue, + &irp->Tail.Overlay.ListEntry); + } + + void CsqRemoveIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + UNREFERENCED_PARAMETER(csq); + RemoveEntryList(&irp->Tail.Overlay.ListEntry); + } + + PIRP CsqPeekNextIrp( + _In_ PIO_CSQ csq, + _In_opt_ PIRP irp, + _In_opt_ PVOID peekContext) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + PLIST_ENTRY entry = irp ? + irp->Tail.Overlay.ListEntry.Flink : + extension->readQueue.Flink; + + while (entry != &extension->readQueue) + { + PIRP candidate = CONTAINING_RECORD(entry, IRP, + Tail.Overlay.ListEntry); + if (!peekContext || + IoGetCurrentIrpStackLocation(candidate)->FileObject == + peekContext) + { + return candidate; + } + entry = entry->Flink; + } + return nullptr; + } + + void CsqAcquireLock(_In_ PIO_CSQ csq, _Out_ PKIRQL irql) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + KeAcquireSpinLock(&extension->readQueueLock, irql); + } + + void CsqReleaseLock(_In_ PIO_CSQ csq, _In_ KIRQL irql) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + KeReleaseSpinLock(&extension->readQueueLock, irql); + } + + void CsqCompleteCanceledIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + auto *context = static_cast( + irp->Tail.Overlay.DriverContext[0]); + irp->Tail.Overlay.DriverContext[0] = nullptr; + if (context) + ExFreePoolWithTag(context, kBrokerReadPoolTag); + InterlockedIncrement64(&g_cancelledReadCount); + CompleteIrp(irp, STATUS_CANCELLED); + ReleasePendingRead(extension); + } + + void SetReadContext( + _Inout_ PIRP irp, + _In_ BrokerReadContext *context) + { + irp->Tail.Overlay.DriverContext[0] = context; + } + + BrokerReadContext *GetReadContext(_In_ PIRP irp) + { + return static_cast( + irp->Tail.Overlay.DriverContext[0]); + } + + void FreeReadContext(_Inout_ PIRP irp) + { + BrokerReadContext *context = GetReadContext(irp); + irp->Tail.Overlay.DriverContext[0] = nullptr; + if (context) + ExFreePoolWithTag(context, kBrokerReadPoolTag); + } + + void CancelQueuedReads( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _In_ NTSTATUS status) + { + PIRP irp; + while ((irp = IoCsqRemoveNextIrp(&extension->readCsq, + fileObject)) != nullptr) + { + FreeReadContext(irp); + InterlockedIncrement64(&g_cancelledReadCount); + CompleteIrp(irp, status); + ReleasePendingRead(extension); + } + } + + void DrainQueuedReads( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _In_ NTSTATUS status) + { + CancelQueuedReads(extension, fileObject, status); + while (HasPendingReads(extension)) + { + KeWaitForSingleObject(&extension->readsDrainedEvent, + Executive, KernelMode, FALSE, nullptr); + } + } + + void QueueReadService() + { + if (!ExAcquireRundownProtection(&g_controlServiceRundown)) + return; + + if (LoadControlDevice() && + InterlockedCompareExchange(&g_retiring, 0, 0) == 0) + { + KeInsertQueueDpc(&g_readServiceDpc, nullptr, nullptr); + } + ExReleaseRundownProtection(&g_controlServiceRundown); + } + AdapterSnapshot GetAdapterSnapshot() { AdapterSnapshot snapshot{}; + if (!ExAcquireRundownProtection(&g_adapterRundown)) + { + snapshot.generation = ReadCounter(&g_adapterGeneration); + snapshot.eventSequence = ReadCounter(&g_eventSequence); + return snapshot; + } + + CAudioAdapter *adapter; KIRQL oldIrql; KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); - snapshot.present = g_adapter != nullptr; + adapter = g_adapter; + snapshot.present = adapter != nullptr; snapshot.generation = ReadCounter(&g_adapterGeneration); snapshot.eventSequence = ReadCounter(&g_eventSequence); KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + + if (adapter) + { + adapter->GetStreamSnapshot(EndpointKind::Render, + &snapshot.render); + adapter->GetStreamSnapshot(EndpointKind::Capture, + &snapshot.capture); + } + ExReleaseRundownProtection(&g_adapterRundown); return snapshot; } @@ -146,16 +392,83 @@ namespace extension->file.signature == kBrokerFileSignature; } - bool UnbindFileLocked(_In_ BrokerControlExtension *extension) + NTSTATUS AcquireBinding( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _Out_ BindingSnapshot *snapshot) + { + RtlZeroMemory(snapshot, sizeof(*snapshot)); + if (!ExAcquireRundownProtection(&extension->bindingRundown)) + return STATUS_DEVICE_NOT_READY; + + NTSTATUS status = STATUS_SUCCESS; + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!IsOwnedFile(extension, fileObject)) + status = STATUS_INVALID_HANDLE; + else if (!extension->file.bound || !extension->file.adapter) + status = STATUS_DEVICE_NOT_READY; + else + { + snapshot->adapter = extension->file.adapter; + snapshot->adapterGeneration = + extension->file.adapterGeneration; + snapshot->bindingGeneration = + extension->file.bindingGeneration; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (!NT_SUCCESS(status)) + ExReleaseRundownProtection(&extension->bindingRundown); + return status; + } + + bool IsCurrentAdapter(_In_ const BindingSnapshot &binding) + { + bool current; + KIRQL oldIrql; + KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); + current = g_adapter == binding.adapter && + ReadCounter(&g_adapterGeneration) == + binding.adapterGeneration; + KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + return current; + } + + CAudioAdapter *UnbindFileLocked( + _In_ BrokerControlExtension *extension) { if (!extension->file.bound) - return false; + return nullptr; + CAudioAdapter *adapter = extension->file.adapter; extension->file.bound = false; extension->file.adapterGeneration = 0; extension->file.bindingGeneration = 0; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = nullptr; + InterlockedCompareExchangePointer( + reinterpret_cast(&g_boundAdapter), + nullptr, adapter); InterlockedIncrement64(&g_bindingGeneration); - return true; + return adapter; + } + + void FinishUnbind( + _In_ BrokerControlExtension *extension, + _In_ CAudioAdapter *adapter) + { + if (!adapter) + return; + + adapter->SetBrokerActive(false); + ExWaitForRundownProtectionRelease(&extension->bindingRundown); + extension->bindingRundownCompleted = true; + adapter->Release(); + + InterlockedIncrement64(&g_unbindCount); + IncrementEventSequence(); } _IRQL_requires_max_(PASSIVE_LEVEL) @@ -171,19 +484,17 @@ namespace BrokerControlExtension *extension = GetControlExtension(controlDevice); - bool unbound; + CAudioAdapter *adapter; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); - unbound = UnbindFileLocked(extension); + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); + FinishUnbind(extension, adapter); + DrainQueuedReads(extension, nullptr, STATUS_DEVICE_NOT_CONNECTED); + ExReleaseFastMutex(&extension->bindingLock); ExReleaseFastMutex(&g_controlLifecycleLock); - - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } } void DeleteSymbolicLinkLocked() @@ -197,6 +508,27 @@ namespace g_linkCreated = false; } + void StartControlServiceLocked() + { + if (!g_controlServiceRundownCompleted) + return; + + KeRemoveQueueDpc(&g_readServiceDpc); + ExReInitializeRundownProtection(&g_controlServiceRundown); + g_controlServiceRundownCompleted = false; + } + + void StopControlServiceLocked() + { + if (g_controlServiceRundownCompleted) + return; + + KeRemoveQueueDpc(&g_readServiceDpc); + ExWaitForRundownProtectionRelease(&g_controlServiceRundown); + KeRemoveQueueDpc(&g_readServiceDpc); + g_controlServiceRundownCompleted = true; + } + NTSTATUS CreateControlDeviceLocked() { if (LoadControlDevice()) @@ -229,9 +561,27 @@ namespace GetControlExtension(deviceObject); RtlZeroMemory(extension, sizeof(*extension)); KeInitializeSpinLock(&extension->stateLock); + ExInitializeFastMutex(&extension->bindingLock); + ExInitializeFastMutex(&extension->captureWriteLock); + ExInitializeRundownProtection(&extension->bindingRundown); + KeInitializeSpinLock(&extension->readQueueLock); + InitializeListHead(&extension->readQueue); + KeInitializeSpinLock(&extension->pendingReadLock); + KeInitializeEvent(&extension->readsDrainedEvent, + NotificationEvent, TRUE); + + status = IoCsqInitialize(&extension->readCsq, + CsqInsertIrp, CsqRemoveIrp, CsqPeekNextIrp, + CsqAcquireLock, CsqReleaseLock, CsqCompleteCanceledIrp); + if (!NT_SUCCESS(status)) + { + IoDeleteDevice(deviceObject); + return status; + } deviceObject->Flags |= DO_DIRECT_IO; + StartControlServiceLocked(); InterlockedExchange(&g_retiring, 0); const PDEVICE_OBJECT previous = static_cast( InterlockedCompareExchangePointer( @@ -285,7 +635,7 @@ namespace KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (InterlockedCompareExchange(&g_retiring, 0, 0) != 0 && - !extension->owner) + !extension->owner && !HasPendingReads(extension)) { const PDEVICE_OBJECT removed = ExchangeControlDevice(nullptr); NT_ASSERT(removed == controlDevice); @@ -325,6 +675,9 @@ namespace extension->file.bound = false; extension->file.adapterGeneration = 0; extension->file.bindingGeneration = 0; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = nullptr; fileObject->FsContext = &extension->file; } KeReleaseSpinLock(&extension->stateLock, oldIrql); @@ -341,20 +694,22 @@ namespace PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); BrokerControlExtension *extension = GetControlExtension(deviceObject); - bool unbound = false; + CAudioAdapter *adapter = nullptr; + bool owned; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); - if (IsOwnedFile(extension, stack->FileObject)) - unbound = UnbindFileLocked(extension); + owned = IsOwnedFile(extension, stack->FileObject); + if (owned) + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); + FinishUnbind(extension, adapter); + if (owned) + DrainQueuedReads(extension, stack->FileObject, STATUS_CANCELLED); + ExReleaseFastMutex(&extension->bindingLock); InterlockedIncrement64(&g_cleanupCount); - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } return CompleteIrp(irp, STATUS_SUCCESS); } @@ -366,25 +721,31 @@ namespace PFILE_OBJECT fileObject = stack->FileObject; BrokerControlExtension *extension = GetControlExtension(deviceObject); - bool unbound = false; + CAudioAdapter *adapter = nullptr; + bool owned; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + owned = IsOwnedFile(extension, fileObject); + if (owned) + adapter = UnbindFileLocked(extension); + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + FinishUnbind(extension, adapter); + if (owned) + DrainQueuedReads(extension, fileObject, STATUS_CANCELLED); + KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (IsOwnedFile(extension, fileObject)) { - unbound = UnbindFileLocked(extension); extension->owner = nullptr; extension->file.signature = 0; if (fileObject) fileObject->FsContext = nullptr; } KeReleaseSpinLock(&extension->stateLock, oldIrql); - - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } + ExReleaseFastMutex(&extension->bindingLock); const NTSTATUS status = CompleteIrp(irp, STATUS_SUCCESS); @@ -413,6 +774,75 @@ namespace header->version == LGAUDIO_BROKER_ABI_VERSION; } + bool ValidatePacketFormat( + _In_ const LGAudioBrokerFormat &format, + _Out_opt_ FormatFingerprint *fingerprint) + { + if (!format.sampleRate || + format.sampleRate > kMaximumSampleRate || + !format.channels || format.channels > 8 || + (format.containerBits != 16 && format.containerBits != 24) || + format.validBits != format.containerBits || + format.blockAlign != + format.channels * (format.containerBits / 8)) + { + return false; + } + + if (fingerprint) + { + fingerprint->sampleRate = format.sampleRate; + fingerprint->channelMask = format.channelMask; + fingerprint->channels = format.channels; + fingerprint->containerBits = format.containerBits; + fingerprint->validBits = format.validBits; + fingerprint->blockAlign = format.blockAlign; + } + return true; + } + + void CopyBrokerFormat( + _In_ const FormatFingerprint &source, + _Out_ LGAudioBrokerFormat *destination) + { + destination->sampleRate = source.sampleRate; + destination->channelMask = source.channelMask; + destination->channels = source.channels; + destination->containerBits = source.containerBits; + destination->validBits = source.validBits; + destination->blockAlign = source.blockAlign; + } + + UINT32 GetBrokerStreamState(_In_ KSSTATE state) + { + switch (state) + { + case KSSTATE_STOP: + return LGAUDIO_BROKER_STREAM_STOPPED; + case KSSTATE_ACQUIRE: + return LGAUDIO_BROKER_STREAM_ACQUIRED; + case KSSTATE_PAUSE: + return LGAUDIO_BROKER_STREAM_PAUSED; + case KSSTATE_RUN: + return LGAUDIO_BROKER_STREAM_RUNNING; + default: + return LGAUDIO_BROKER_STREAM_UNAVAILABLE; + } + } + + void FillBrokerStreamState( + _In_ const AudioStreamSnapshot &source, + _Out_ LGAudioBrokerStreamState *destination) + { + destination->generation = source.generation; + destination->state = GetBrokerStreamState(source.state); + if (source.formatValid) + { + destination->flags |= LGAUDIO_BROKER_STREAM_FORMAT_VALID; + CopyBrokerFormat(source.format, &destination->format); + } + } + NTSTATUS QueryCaps( _Inout_ PIRP irp, _In_ ULONG outputLength) @@ -428,7 +858,11 @@ namespace caps->version = LGAUDIO_BROKER_ABI_VERSION; caps->minimumVersion = LGAUDIO_BROKER_ABI_VERSION; caps->maximumVersion = LGAUDIO_BROKER_ABI_VERSION; - caps->capabilities = LGAUDIO_BROKER_CAP_CONTROL; + caps->capabilities = LGAUDIO_BROKER_CAP_CONTROL | + LGAUDIO_BROKER_CAP_RENDER_READ | + LGAUDIO_BROKER_CAP_CAPTURE_WRITE; + caps->maximumRenderTransfer = LGAUDIO_BROKER_MAX_PAYLOAD_BYTES; + caps->maximumCaptureTransfer = LGAUDIO_BROKER_MAX_PAYLOAD_BYTES; return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*caps)); } @@ -452,7 +886,8 @@ namespace if (IsOwnedFile(extension, fileObject)) { bound = extension->file.bound && snapshot.present && - extension->file.adapterGeneration == snapshot.generation; + extension->file.adapterGeneration == snapshot.generation && + extension->file.adapter == LoadBoundAdapter(); } KeReleaseSpinLock(&extension->stateLock, oldIrql); @@ -464,12 +899,22 @@ namespace state->adapterGeneration = snapshot.generation; state->eventSequence = snapshot.eventSequence; state->bindingGeneration = bindingGeneration; - state->render.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; - state->capture.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; if (snapshot.present) + { state->flags |= LGAUDIO_BROKER_STATE_ADAPTER_PRESENT; + FillBrokerStreamState(snapshot.render, &state->render); + FillBrokerStreamState(snapshot.capture, &state->capture); + } + else + { + state->render.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; + state->capture.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; + } if (bound) - state->flags |= LGAUDIO_BROKER_STATE_BOUND; + { + state->flags |= LGAUDIO_BROKER_STATE_BOUND | + LGAUDIO_BROKER_STATE_DATA_PLANE_ACTIVE; + } return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*state)); } @@ -506,22 +951,43 @@ namespace status = STATUS_RETRY; else { + ExAcquireFastMutex(&extension->bindingLock); + if (extension->bindingRundownCompleted) + { + ExReInitializeRundownProtection(&extension->bindingRundown); + extension->bindingRundownCompleted = false; + } + KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (!IsOwnedFile(extension, fileObject)) status = STATUS_INVALID_HANDLE; else if (extension->file.bound) status = STATUS_INVALID_DEVICE_STATE; - else + else if (LoadBoundAdapter()) + status = STATUS_DEVICE_BUSY; + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (NT_SUCCESS(status)) { const ULONGLONG bindingGeneration = static_cast( InterlockedIncrement64(&g_bindingGeneration)); + adapter->AddRef(); + adapter->SetBrokerActive(true); + InterlockedExchangePointer( + reinterpret_cast(&g_boundAdapter), adapter); + + KeAcquireSpinLock(&extension->stateLock, &oldIrql); extension->file.bound = true; extension->file.adapterGeneration = generation; extension->file.bindingGeneration = bindingGeneration; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = adapter; bind->bindingGeneration = bindingGeneration; + KeReleaseSpinLock(&extension->stateLock, oldIrql); } - KeReleaseSpinLock(&extension->stateLock, oldIrql); + ExReleaseFastMutex(&extension->bindingLock); } BrokerDeviceReleaseAdapter(); @@ -545,21 +1011,20 @@ namespace sizeof(LGAudioBrokerHeader))) return CompleteIrp(irp, STATUS_INVALID_PARAMETER); - bool unbound = false; + CAudioAdapter *adapter = nullptr; NTSTATUS status = STATUS_SUCCESS; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (!IsOwnedFile(extension, fileObject)) status = STATUS_INVALID_HANDLE; else - unbound = UnbindFileLocked(extension); + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } + FinishUnbind(extension, adapter); + DrainQueuedReads(extension, fileObject, STATUS_CANCELLED); + ExReleaseFastMutex(&extension->bindingLock); return CompleteIrp(irp, status); } @@ -584,9 +1049,401 @@ namespace stats->cleanupCount = ReadCounter(&g_cleanupCount); stats->rejectedRequestCount = ReadCounter(&g_rejectedRequestCount); + stats->renderPacketCount = ReadCounter(&g_renderPacketCount); + stats->renderByteCount = ReadCounter(&g_renderByteCount); + stats->renderDropBytes = ReadCounter(&g_renderDropBytes); + stats->capturePacketCount = ReadCounter(&g_capturePacketCount); + stats->captureByteCount = ReadCounter(&g_captureByteCount); + stats->captureDropBytes = ReadCounter(&g_captureDropBytes); + stats->captureUnderrunBytes = + ReadCounter(&g_captureUnderrunBytes); + stats->cancelledReadCount = ReadCounter(&g_cancelledReadCount); return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*stats)); } + void *MapDirectBuffer( + _In_ PIRP irp, + _In_ ULONG requiredBytes) + { + if (!irp->MdlAddress || + MmGetMdlByteCount(irp->MdlAddress) < requiredBytes) + { + return nullptr; + } + + return MmGetSystemAddressForMdlSafe(irp->MdlAddress, + NormalPagePriority | MdlMappingNoExecute); + } + + void CompletePendingRead( + _In_ BrokerControlExtension *extension, + _Inout_ PIRP irp, + _In_ NTSTATUS status, + _In_ ULONG_PTR information = 0) + { + FreeReadContext(irp); + if (status == STATUS_CANCELLED || + status == STATUS_DEVICE_NOT_READY || + status == STATUS_DEVICE_NOT_CONNECTED || + status == STATUS_RETRY) + { + InterlockedIncrement64(&g_cancelledReadCount); + } + CompleteIrp(irp, status, information); + ReleasePendingRead(extension); + } + + NTSTATUS DispatchRead( + _In_ PDEVICE_OBJECT deviceObject, + _Inout_ PIRP irp) + { + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + const ULONG length = stack->Parameters.Read.Length; + if (length <= sizeof(LGAudioBrokerPacketHeader) || + length > sizeof(LGAudioBrokerPacketHeader) + + LGAUDIO_BROKER_MAX_PAYLOAD_BYTES || + !MapDirectBuffer(irp, length)) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + BrokerControlExtension *extension = + GetControlExtension(deviceObject); + BindingSnapshot binding{}; + NTSTATUS status = AcquireBinding(extension, stack->FileObject, + &binding); + if (!NT_SUCCESS(status)) + return CompleteIrp(irp, status); + + AudioStreamSnapshot render{}; + binding.adapter->GetStreamSnapshot(EndpointKind::Render, &render); + const ULONG payloadCapacity = + length - sizeof(LGAudioBrokerPacketHeader); + if (!IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + else if (!render.formatValid || render.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (!render.format.blockAlign || + payloadCapacity < render.format.blockAlign) + status = STATUS_INVALID_PARAMETER; + + if (!NT_SUCCESS(status)) + { + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, status); + } + +#pragma warning(push) +#pragma warning(disable:4996) // Required for pre-2004 Windows 10 releases. + auto *context = static_cast( + ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(BrokerReadContext), + kBrokerReadPoolTag)); +#pragma warning(pop) + if (!context) + { + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, STATUS_INSUFFICIENT_RESOURCES); + } + + context->adapterGeneration = binding.adapterGeneration; + context->bindingGeneration = binding.bindingGeneration; + context->streamGeneration = render.generation; + SetReadContext(irp, context); + + const LONG pending = AddPendingRead(extension); + if (pending > kMaximumPendingReads) + { + ReleasePendingRead(extension); + FreeReadContext(irp); + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, STATUS_DEVICE_BUSY); + } + + IoMarkIrpPending(irp); + IoCsqInsertIrp(&extension->readCsq, irp, nullptr); + ExReleaseRundownProtection(&extension->bindingRundown); + QueueReadService(); + return STATUS_PENDING; + } + + bool ServiceReadIrp( + _In_ BrokerControlExtension *extension, + _Inout_ PIRP irp) + { + BrokerReadContext *context = GetReadContext(irp); + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + if (!context || + !ExAcquireRundownProtection(&extension->bindingRundown)) + { + CompletePendingRead(extension, irp, STATUS_DEVICE_NOT_READY); + return true; + } + + BindingSnapshot binding{}; + NTSTATUS status = STATUS_SUCCESS; + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!IsOwnedFile(extension, stack->FileObject) || + !extension->file.bound || !extension->file.adapter) + { + status = STATUS_DEVICE_NOT_READY; + } + else + { + binding.adapter = extension->file.adapter; + binding.adapterGeneration = extension->file.adapterGeneration; + binding.bindingGeneration = extension->file.bindingGeneration; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (NT_SUCCESS(status) && + (binding.adapterGeneration != context->adapterGeneration || + binding.bindingGeneration != context->bindingGeneration)) + { + status = STATUS_RETRY; + } + if (NT_SUCCESS(status) && !IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + + AudioStreamSnapshot render{}; + if (NT_SUCCESS(status)) + { + binding.adapter->GetStreamSnapshot(EndpointKind::Render, &render); + if (!render.formatValid || render.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (render.generation != context->streamGeneration) + status = STATUS_RETRY; + } + + const ULONG length = stack->Parameters.Read.Length; + auto *buffer = static_cast(MapDirectBuffer(irp, length)); + if (NT_SUCCESS(status) && !buffer) + status = STATUS_INSUFFICIENT_RESOURCES; + + ULONG bytesRead = 0; + bool discontinuity = false; + if (NT_SUCCESS(status)) + { + const ULONG capacity = + length - sizeof(LGAudioBrokerPacketHeader); + BYTE *payload = buffer + sizeof(LGAudioBrokerPacketHeader); + status = binding.adapter->ReadRenderFrames(render.format, + render.generation, payload, capacity, &bytesRead, + &discontinuity); + } + + if (NT_SUCCESS(status) && !bytesRead) + { + IoCsqInsertIrp(&extension->readCsq, irp, nullptr); + ExReleaseRundownProtection(&extension->bindingRundown); + return false; + } + + if (NT_SUCCESS(status)) + { + LGAudioBrokerPacketHeader packet{}; + packet.size = sizeof(packet); + packet.version = LGAUDIO_BROKER_ABI_VERSION; + packet.flags = discontinuity ? + LGAUDIO_BROKER_PACKET_DISCONTINUITY : 0; + packet.direction = LGAUDIO_BROKER_PACKET_RENDER; + packet.adapterGeneration = binding.adapterGeneration; + packet.bindingGeneration = binding.bindingGeneration; + packet.streamGeneration = render.generation; + packet.sequence = static_cast( + InterlockedIncrement64(&g_renderSequence) - 1); + CopyBrokerFormat(render.format, &packet.format); + packet.frameCount = bytesRead / render.format.blockAlign; + packet.payloadBytes = bytesRead; + RtlCopyMemory(buffer, &packet, sizeof(packet)); + + InterlockedIncrement64(&g_renderPacketCount); + AddCounter(&g_renderByteCount, bytesRead); + } + + const ULONG_PTR information = NT_SUCCESS(status) ? + sizeof(LGAudioBrokerPacketHeader) + bytesRead : 0; + CompletePendingRead(extension, irp, status, information); + ExReleaseRundownProtection(&extension->bindingRundown); + return true; + } + + void ReadServiceDpc( + _In_ PKDPC dpc, + _In_opt_ PVOID deferredContext, + _In_opt_ PVOID systemArgument1, + _In_opt_ PVOID systemArgument2) + { + UNREFERENCED_PARAMETER(dpc); + UNREFERENCED_PARAMETER(deferredContext); + UNREFERENCED_PARAMETER(systemArgument1); + UNREFERENCED_PARAMETER(systemArgument2); + + if (!ExAcquireRundownProtection(&g_controlServiceRundown)) + return; + + PDEVICE_OBJECT controlDevice = LoadControlDevice(); + ULONG serviced = 0; + if (controlDevice) + { + BrokerControlExtension *extension = + GetControlExtension(controlDevice); + while (serviced < kReadServiceBatch) + { + PIRP irp = IoCsqRemoveNextIrp(&extension->readCsq, nullptr); + if (!irp) + break; + if (!ServiceReadIrp(extension, irp)) + break; + ++serviced; + } + + if (serviced == kReadServiceBatch && + HasPendingReads(extension)) + { + KeInsertQueueDpc(&g_readServiceDpc, nullptr, nullptr); + } + } + + ExReleaseRundownProtection(&g_controlServiceRundown); + } + + NTSTATUS DispatchWrite( + _In_ PDEVICE_OBJECT deviceObject, + _Inout_ PIRP irp) + { + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + const ULONG length = stack->Parameters.Write.Length; + if (length <= sizeof(LGAudioBrokerPacketHeader) || + length > sizeof(LGAudioBrokerPacketHeader) + + LGAUDIO_BROKER_MAX_PAYLOAD_BYTES) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + const auto *mapped = static_cast( + MapDirectBuffer(irp, length)); + if (!mapped) + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + + LGAudioBrokerPacketHeader packet{}; + RtlCopyMemory(&packet, mapped, sizeof(packet)); + FormatFingerprint format{}; + const UINT32 allowedFlags = + LGAUDIO_BROKER_PACKET_DISCONTINUITY; + if (packet.size != sizeof(packet) || + packet.version != LGAUDIO_BROKER_ABI_VERSION || + (packet.flags & ~allowedFlags) != 0 || + packet.direction != LGAUDIO_BROKER_PACKET_CAPTURE || + packet.reserved[0] || packet.reserved[1] || + !packet.payloadBytes || + packet.payloadBytes > LGAUDIO_BROKER_MAX_PAYLOAD_BYTES || + packet.payloadBytes != length - sizeof(packet) || + !ValidatePacketFormat(packet.format, &format) || + !packet.frameCount || + (packet.payloadBytes % format.blockAlign) != 0 || + packet.frameCount != packet.payloadBytes / format.blockAlign) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + BrokerControlExtension *extension = + GetControlExtension(deviceObject); + ExAcquireFastMutex(&extension->captureWriteLock); + + BindingSnapshot binding{}; + NTSTATUS status = AcquireBinding(extension, stack->FileObject, + &binding); + if (NT_SUCCESS(status) && + (packet.adapterGeneration != binding.adapterGeneration || + packet.bindingGeneration != binding.bindingGeneration)) + { + status = STATUS_RETRY; + } + if (NT_SUCCESS(status) && !IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + + AudioStreamSnapshot capture{}; + if (NT_SUCCESS(status)) + { + binding.adapter->GetStreamSnapshot(EndpointKind::Capture, + &capture); + if (!capture.formatValid || capture.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (packet.streamGeneration != capture.generation || + !EqualFormatFingerprint(format, capture.format)) + status = STATUS_RETRY; + } + + if (NT_SUCCESS(status)) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!extension->file.bound || + extension->file.adapter != binding.adapter || + extension->file.bindingGeneration != + binding.bindingGeneration) + { + status = STATUS_DEVICE_NOT_READY; + } + else if (extension->file.captureStreamGeneration != + capture.generation) + { + if (packet.sequence != 0) + status = STATUS_RETRY; + } + else if (packet.sequence != + extension->file.nextCaptureSequence) + { + status = STATUS_RETRY; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + } + + ULONG droppedBytes = 0; + if (NT_SUCCESS(status)) + { + const BYTE *payload = + mapped + sizeof(LGAudioBrokerPacketHeader); + status = binding.adapter->WriteCaptureFrames(format, + capture.generation, + (packet.flags & LGAUDIO_BROKER_PACKET_DISCONTINUITY) != 0, + payload, packet.payloadBytes, &droppedBytes); + } + + if (NT_SUCCESS(status)) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!extension->file.bound || + extension->file.bindingGeneration != + binding.bindingGeneration) + { + status = STATUS_DEVICE_NOT_READY; + } + else + { + extension->file.captureStreamGeneration = capture.generation; + extension->file.nextCaptureSequence = packet.sequence + 1; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + } + + if (NT_SUCCESS(status)) + { + InterlockedIncrement64(&g_capturePacketCount); + AddCounter(&g_captureByteCount, packet.payloadBytes); + AddCounter(&g_captureDropBytes, droppedBytes); + } + + ExReleaseFastMutex(&extension->captureWriteLock); + const NTSTATUS completedStatus = CompleteIrp(irp, status, + NT_SUCCESS(status) ? length : 0); + if (binding.adapter) + ExReleaseRundownProtection(&extension->bindingRundown); + return completedStatus; + } + NTSTATUS DispatchDeviceControl( _In_ PDEVICE_OBJECT deviceObject, _Inout_ PIRP irp) @@ -659,6 +1516,8 @@ NTSTATUS BrokerDeviceInitialize(_In_ PDRIVER_OBJECT driverObject) ExInitializeFastMutex(&g_adapterLifecycleLock); KeInitializeSpinLock(&g_adapterStateLock); ExInitializeRundownProtection(&g_adapterRundown); + ExInitializeRundownProtection(&g_controlServiceRundown); + KeInitializeDpc(&g_readServiceDpc, ReadServiceDpc, nullptr); g_driverObject = driverObject; ExAcquireFastMutex(&g_controlLifecycleLock); @@ -699,6 +1558,7 @@ void BrokerDeviceShutdown() ExAcquireFastMutex(&g_controlLifecycleLock); SetControlRetiringLocked(true); DeleteSymbolicLinkLocked(); + StopControlServiceLocked(); deleteDevice = TakeRetiredControlDeviceLocked(); NT_ASSERT(deleteDevice || !LoadControlDevice()); ExReleaseFastMutex(&g_controlLifecycleLock); @@ -741,9 +1601,20 @@ NTSTATUS BrokerDeviceDispatch( return CompleteIrp(irp, STATUS_SUCCESS); case IRP_MJ_READ: + { + const NTSTATUS status = DispatchRead(deviceObject, irp); + if (!NT_SUCCESS(status)) + InterlockedIncrement64(&g_rejectedRequestCount); + return status; + } + case IRP_MJ_WRITE: - InterlockedIncrement64(&g_rejectedRequestCount); - return CompleteIrp(irp, STATUS_NOT_SUPPORTED); + { + const NTSTATUS status = DispatchWrite(deviceObject, irp); + if (!NT_SUCCESS(status)) + InterlockedIncrement64(&g_rejectedRequestCount); + return status; + } default: InterlockedIncrement64(&g_rejectedRequestCount); @@ -778,6 +1649,7 @@ void BrokerDeviceNotifyPnpDeviceAdded() if (NT_SUCCESS(status)) { g_linkCreated = true; + StartControlServiceLocked(); SetControlRetiringLocked(false); } else @@ -803,6 +1675,7 @@ void BrokerDeviceNotifyPnpDeviceRemoved() ExAcquireFastMutex(&g_controlLifecycleLock); SetControlRetiringLocked(true); DeleteSymbolicLinkLocked(); + StopControlServiceLocked(); deleteDevice = TakeRetiredControlDeviceLocked(); ExReleaseFastMutex(&g_controlLifecycleLock); @@ -923,3 +1796,44 @@ void BrokerDeviceReleaseAdapter() { ExReleaseRundownProtection(&g_adapterRundown); } + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyStreamChanged( + _In_ CAudioAdapter *adapter, + _In_ EndpointKind kind) +{ + bool current; + KIRQL oldIrql; + KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); + current = adapter && g_adapter == adapter; + KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + if (!current) + return; + + IncrementEventSequence(); + if (kind == EndpointKind::Render) + QueueReadService(); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyRenderAvailable( + _In_ CAudioAdapter *adapter, + _In_ ULONG bytes, + _In_ ULONG droppedBytes) +{ + UNREFERENCED_PARAMETER(bytes); + if (!adapter || LoadBoundAdapter() != adapter) + return; + + AddCounter(&g_renderDropBytes, droppedBytes); + QueueReadService(); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceRecordCaptureUnderrun( + _In_ CAudioAdapter *adapter, + _In_ ULONGLONG bytes) +{ + if (adapter && LoadBoundAdapter() == adapter) + AddCounter(&g_captureUnderrunBytes, bytes); +} diff --git a/idd/LGAudio/BrokerDevice.h b/idd/LGAudio/BrokerDevice.h index f78aaaa3..43a3c0d2 100644 --- a/idd/LGAudio/BrokerDevice.h +++ b/idd/LGAudio/BrokerDevice.h @@ -21,6 +21,7 @@ #pragma once #include "Common.h" +#include "Formats.h" class CAudioAdapter; @@ -75,3 +76,19 @@ CAudioAdapter *BrokerDeviceAcquireAdapter( _IRQL_requires_max_(DISPATCH_LEVEL) void BrokerDeviceReleaseAdapter(); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyStreamChanged( + _In_ CAudioAdapter *adapter, + _In_ EndpointKind kind); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyRenderAvailable( + _In_ CAudioAdapter *adapter, + _In_ ULONG bytes, + _In_ ULONG droppedBytes); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceRecordCaptureUnderrun( + _In_ CAudioAdapter *adapter, + _In_ ULONGLONG bytes); diff --git a/idd/LGAudio/MiniportWaveRTStream.cpp b/idd/LGAudio/MiniportWaveRTStream.cpp index c4ed955e..445c9c4c 100644 --- a/idd/LGAudio/MiniportWaveRTStream.cpp +++ b/idd/LGAudio/MiniportWaveRTStream.cpp @@ -114,8 +114,7 @@ NTSTATUS CMiniportWaveRTStream::Initialize( m_format = *format; m_fingerprint = fingerprint; - if (m_kind == EndpointKind::Render) - status = m_adapter->ConfigureRender(m_fingerprint); + status = m_adapter->ConfigureStream(m_kind, m_fingerprint); return status; } @@ -195,8 +194,7 @@ NTSTATUS CMiniportWaveRTStream::SetFormat( m_fingerprint = fingerprint; KeReleaseSpinLock(&m_positionLock, oldIrql); - if (m_kind == EndpointKind::Render) - status = m_adapter->ConfigureRender(m_fingerprint); + status = m_adapter->ConfigureStream(m_kind, m_fingerprint); return status; } @@ -244,6 +242,8 @@ NTSTATUS CMiniportWaveRTStream::SetState(_In_ KSSTATE state) if (resetCapture) m_adapter->ResetCapture(); + m_adapter->SetStreamState(m_kind, KSSTATE_RUN); + KeAcquireSpinLock(&m_positionLock, &oldIrql); if (m_state == KSSTATE_RUN) ArmTimerLocked(); @@ -280,6 +280,7 @@ NTSTATUS CMiniportWaveRTStream::SetState(_In_ KSSTATE state) KeReleaseSpinLock(&m_positionLock, oldIrql); StopTimer(); + m_adapter->SetStreamState(m_kind, state); return STATUS_SUCCESS; } @@ -901,6 +902,7 @@ void CMiniportWaveRTStream::AdvanceEventPacketsLocked( { m_eosNotificationSent = true; m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); m_pendingBytes = 0; m_frameRemainder = 0; break; @@ -944,6 +946,7 @@ void CMiniportWaveRTStream::TransferFramesLocked(_In_ ULONGLONG bytes) if (m_linearPosition >= m_eosLinearPosition) { m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); return; } @@ -997,7 +1000,10 @@ void CMiniportWaveRTStream::TransferFramesLocked(_In_ ULONGLONG bytes) if (m_kind == EndpointKind::Render && m_eosReceived && m_linearPosition >= m_eosLinearPosition) + { m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); + } } void CMiniportWaveRTStream::SignalNotificationsLocked() diff --git a/idd/LGCommon/LGAudioBroker.h b/idd/LGCommon/LGAudioBroker.h index 9dff30fc..1ab7f380 100644 --- a/idd/LGCommon/LGAudioBroker.h +++ b/idd/LGCommon/LGAudioBroker.h @@ -44,11 +44,12 @@ static const GUID LGAUDIO_BROKER_DEVICE_CLASS_GUID = { 0xb4, 0x90, 0x28, 0xc5, 0x30, 0xd2, 0x38, 0xcd } }; static constexpr UINT32 LGAUDIO_BROKER_ABI_VERSION = 1U; +// The transfer limits in LGAudioBrokerCaps exclude the packet header. +static constexpr UINT32 LGAUDIO_BROKER_MAX_PAYLOAD_BYTES = 64U * 1024U; enum LGAudioBrokerCapability : UINT32 { LGAUDIO_BROKER_CAP_CONTROL = 1U << 0, - // These remain clear until the corresponding data plane is implemented. LGAUDIO_BROKER_CAP_RENDER_READ = 1U << 1, LGAUDIO_BROKER_CAP_CAPTURE_WRITE = 1U << 2, LGAUDIO_BROKER_CAP_CLOCK_CONTROL = 1U << 3 @@ -70,6 +71,22 @@ enum LGAudioBrokerStreamStateValue : UINT32 LGAUDIO_BROKER_STREAM_RUNNING = 4 }; +enum LGAudioBrokerStreamFlag : UINT32 +{ + LGAUDIO_BROKER_STREAM_FORMAT_VALID = 1U << 0 +}; + +enum LGAudioBrokerPacketDirection : UINT32 +{ + LGAUDIO_BROKER_PACKET_RENDER = 1, + LGAUDIO_BROKER_PACKET_CAPTURE = 2 +}; + +enum LGAudioBrokerPacketFlag : UINT32 +{ + LGAUDIO_BROKER_PACKET_DISCONTINUITY = 1U << 0 +}; + static constexpr UINT32 IOCTL_LGAUDIO_BROKER_QUERY_CAPS = CTL_CODE(FILE_DEVICE_UNKNOWN, 0x900, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS); @@ -106,12 +123,33 @@ struct LGAudioBrokerFormat struct LGAudioBrokerStreamState { + // Changes on format configuration and each transition to STOP. UINT64 generation; UINT32 state; UINT32 flags; LGAudioBrokerFormat format; }; +struct LGAudioBrokerPacketHeader +{ + // READ returns this header followed by render payload. WRITE accepts this + // header followed by capture payload. size is always the header size. + UINT32 size; + UINT32 version; + UINT32 flags; + UINT32 direction; + UINT64 adapterGeneration; + UINT64 bindingGeneration; + UINT64 streamGeneration; + // Render sequence is monotonic for the driver lifetime. Capture starts at + // zero for each binding/stream-generation pair and must be contiguous. + UINT64 sequence; + LGAudioBrokerFormat format; + UINT32 frameCount; + UINT32 payloadBytes; + UINT32 reserved[2]; +}; + struct LGAudioBrokerCaps { UINT32 size; @@ -160,7 +198,15 @@ struct LGAudioBrokerStats UINT64 unbindCount; UINT64 cleanupCount; UINT64 rejectedRequestCount; - UINT64 reserved[2]; + UINT64 renderPacketCount; + UINT64 renderByteCount; + UINT64 renderDropBytes; + UINT64 capturePacketCount; + UINT64 captureByteCount; + UINT64 captureDropBytes; + UINT64 captureUnderrunBytes; + UINT64 cancelledReadCount; + UINT64 reserved[4]; }; #pragma pack(pop) @@ -171,11 +217,16 @@ static_assert(sizeof(LGAudioBrokerFormat) == 16, "LGAudio broker format layout changed"); static_assert(sizeof(LGAudioBrokerStreamState) == 32, "LGAudio broker stream-state layout changed"); +static_assert(sizeof(LGAudioBrokerPacketHeader) == 80, + "LGAudio broker packet layout changed"); static_assert(sizeof(LGAudioBrokerCaps) == 64, "LGAudio broker capabilities layout changed"); static_assert(sizeof(LGAudioBrokerState) == 104, "LGAudio broker state layout changed"); static_assert(sizeof(LGAudioBrokerBind) == 48, "LGAudio broker binding layout changed"); -static_assert(sizeof(LGAudioBrokerStats) == 80, +static_assert(sizeof(LGAudioBrokerStats) == 160, "LGAudio broker statistics layout changed"); +static_assert(LGAUDIO_BROKER_MAX_PAYLOAD_BYTES >= + (192000U / 100U) * 8U * 3U, + "LGAudio broker payload cannot hold 10 ms at the maximum format");