From 5e077864685e91bfaa4e8494124ada410a73d531 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Wed, 2 Sep 2026 00:02:57 +1000 Subject: [PATCH] [idd] audio: add the mediated PCM data plane Expose cancel-safe direct-I/O queues for reading rendered frames and writing captured frames through the private broker endpoint. Separate render and capture storage, publish format and state generations, and preserve local loopback while no broker is bound. --- idd/LGAudio/Adapter.cpp | 241 ++++++- idd/LGAudio/Adapter.h | 52 +- idd/LGAudio/BrokerDevice.cpp | 1004 ++++++++++++++++++++++++-- idd/LGAudio/BrokerDevice.h | 17 + idd/LGAudio/MiniportWaveRTStream.cpp | 14 +- idd/LGCommon/LGAudioBroker.h | 57 +- 6 files changed, 1322 insertions(+), 63 deletions(-) diff --git a/idd/LGAudio/Adapter.cpp b/idd/LGAudio/Adapter.cpp index 225cef1c..aec3348c 100644 --- a/idd/LGAudio/Adapter.cpp +++ b/idd/LGAudio/Adapter.cpp @@ -112,33 +112,140 @@ NTSTATUS InstallWaveSubdevice( } CAudioAdapter::CAudioAdapter() : - m_refCount(1) + m_refCount(1), + m_brokerActive(0), + m_renderDiscontinuity(0), + m_renderState{}, + m_captureState{} { + KeInitializeSpinLock(&m_transportLock); + KeInitializeSpinLock(&m_streamStateLock); + m_renderState.state = KSSTATE_STOP; + m_captureState.state = KSSTATE_STOP; } CAudioAdapter::~CAudioAdapter() { - m_ring.Destroy(); + m_captureRing.Destroy(); + m_renderRing.Destroy(); } NTSTATUS CAudioAdapter::Initialize() { NTSTATUS status = m_clock.Initialize(); if (NT_SUCCESS(status)) - status = m_ring.Init(); + status = m_renderRing.Init(); + if (NT_SUCCESS(status)) + status = m_captureRing.Init(); return status; } + _IRQL_requires_max_(DISPATCH_LEVEL) -NTSTATUS CAudioAdapter::ConfigureRender( +NTSTATUS CAudioAdapter::ConfigureStream( + _In_ EndpointKind kind, _In_ const FormatFingerprint &format) { - return m_ring.Configure(format); + CAudioRingBuffer &ring = kind == EndpointKind::Render ? + m_renderRing : m_captureRing; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + const NTSTATUS status = ring.Configure(format); + if (!NT_SUCCESS(status)) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return status; + } + + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + AudioStreamSnapshot &stream = kind == EndpointKind::Render ? + m_renderState : m_captureState; + stream.format = format; + stream.formatValid = true; + ++stream.generation; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); + KeReleaseSpinLock(&m_transportLock, transportIrql); + + BrokerDeviceNotifyStreamChanged(this, kind); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::SetStreamState( + _In_ EndpointKind kind, + _In_ KSSTATE state) +{ + bool changed; + bool reset; + + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + AudioStreamSnapshot &stream = kind == EndpointKind::Render ? + m_renderState : m_captureState; + changed = stream.state != state; + reset = changed && state == KSSTATE_STOP; + stream.state = state; + if (reset) + ++stream.generation; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); + + if (reset) + { + if (kind == EndpointKind::Render) + m_renderRing.Reset(); + else + m_captureRing.Reset(); + } + KeReleaseSpinLock(&m_transportLock, transportIrql); + if (changed) + BrokerDeviceNotifyStreamChanged(this, kind); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::GetStreamSnapshot( + _In_ EndpointKind kind, + _Out_ AudioStreamSnapshot *snapshot) +{ + if (!snapshot) + return; + + KIRQL oldIrql; + KeAcquireSpinLock(&m_streamStateLock, &oldIrql); + *snapshot = kind == EndpointKind::Render ? + m_renderState : m_captureState; + KeReleaseSpinLock(&m_streamStateLock, oldIrql); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void CAudioAdapter::SetBrokerActive(_In_ bool active) +{ + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const LONG value = active ? 1 : 0; + if (InterlockedExchange(&m_brokerActive, value) == value) + { + KeReleaseSpinLock(&m_transportLock, oldIrql); + return; + } + + m_captureRing.Reset(); + m_renderRing.Reset(); + InterlockedExchange(&m_renderDiscontinuity, active ? 1 : 0); + KeReleaseSpinLock(&m_transportLock, oldIrql); } _IRQL_requires_max_(DISPATCH_LEVEL) void CAudioAdapter::ResetCapture() { - m_ring.Reset(); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + if (InterlockedCompareExchange(&m_brokerActive, 0, 0)) + m_captureRing.Reset(); + else + m_renderRing.Reset(); + KeReleaseSpinLock(&m_transportLock, oldIrql); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -146,7 +253,14 @@ void CAudioAdapter::WriteRenderFrames( _In_reads_bytes_(bytes) const BYTE *data, _In_ ULONG bytes) { - m_ring.Push(data, bytes); + ULONG droppedBytes = 0; + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + m_renderRing.Push(data, bytes, &droppedBytes); + if (droppedBytes) + InterlockedExchange(&m_renderDiscontinuity, 1); + KeReleaseSpinLock(&m_transportLock, oldIrql); + BrokerDeviceNotifyRenderAvailable(this, bytes, droppedBytes); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -155,7 +269,16 @@ void CAudioAdapter::ReadCaptureFrames( _Out_writes_bytes_(bytes) BYTE *data, _In_ ULONG bytes) { - m_ring.Pop(format, data, bytes); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const bool brokerActive = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0; + CAudioRingBuffer &ring = brokerActive ? + m_captureRing : m_renderRing; + const ULONG copied = ring.Pop(format, data, bytes); + KeReleaseSpinLock(&m_transportLock, oldIrql); + if (brokerActive && copied < bytes) + BrokerDeviceRecordCaptureUnderrun(this, bytes - copied); } _IRQL_requires_max_(DISPATCH_LEVEL) @@ -163,7 +286,107 @@ void CAudioAdapter::DiscardCaptureFrames( _In_ const FormatFingerprint &format, _In_ ULONGLONG bytes) { - m_ring.Discard(format, bytes); + KIRQL oldIrql; + KeAcquireSpinLock(&m_transportLock, &oldIrql); + const bool brokerActive = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0; + CAudioRingBuffer &ring = brokerActive ? + m_captureRing : m_renderRing; + const ULONG discarded = ring.Discard(format, bytes); + KeReleaseSpinLock(&m_transportLock, oldIrql); + if (brokerActive && bytes > discarded) + BrokerDeviceRecordCaptureUnderrun(this, bytes - discarded); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS CAudioAdapter::ReadRenderFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _Out_writes_bytes_(capacity) BYTE *data, + _In_ ULONG capacity, + _Out_ PULONG bytesRead, + _Out_ bool *discontinuity) +{ + if (!data || !bytesRead || !discontinuity || !format.blockAlign) + return STATUS_INVALID_PARAMETER; + + *bytesRead = 0; + *discontinuity = false; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + + KIRQL stateIrql; + KeAcquireSpinLock(&m_streamStateLock, &stateIrql); + const bool active = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0 && + m_renderState.state == KSSTATE_RUN && + m_renderState.formatValid && + m_renderState.generation == streamGeneration && + EqualFormatFingerprint(m_renderState.format, format); + KeReleaseSpinLock(&m_streamStateLock, stateIrql); + if (!active) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_DEVICE_NOT_READY; + } + + ULONG bytes = m_renderRing.Available(); + if (bytes > capacity) + bytes = capacity; + bytes -= bytes % format.blockAlign; + if (!bytes) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_SUCCESS; + } + + const ULONG copied = m_renderRing.Pop(format, data, bytes); + if (copied) + { + *discontinuity = InterlockedExchange( + &m_renderDiscontinuity, 0) != 0; + } + *bytesRead = copied; + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_SUCCESS; +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +NTSTATUS CAudioAdapter::WriteCaptureFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _In_ bool discontinuity, + _In_reads_bytes_(bytes) const BYTE *data, + _In_ ULONG bytes, + _Out_ PULONG droppedBytes) +{ + if (!data || !droppedBytes) + return STATUS_INVALID_PARAMETER; + + *droppedBytes = 0; + KIRQL transportIrql; + KeAcquireSpinLock(&m_transportLock, &transportIrql); + + KIRQL stateIrql; + KeAcquireSpinLock(&m_streamStateLock, &stateIrql); + const bool active = + InterlockedCompareExchange(&m_brokerActive, 0, 0) != 0 && + m_captureState.state == KSSTATE_RUN && + m_captureState.formatValid && + m_captureState.generation == streamGeneration && + EqualFormatFingerprint(m_captureState.format, format); + KeReleaseSpinLock(&m_streamStateLock, stateIrql); + if (!active) + { + KeReleaseSpinLock(&m_transportLock, transportIrql); + return STATUS_DEVICE_NOT_READY; + } + + if (discontinuity) + m_captureRing.Reset(); + const ULONG written = m_captureRing.Push(data, bytes, droppedBytes); + KeReleaseSpinLock(&m_transportLock, transportIrql); + return written == bytes ? STATUS_SUCCESS : STATUS_INVALID_PARAMETER; } _IRQL_requires_max_(DISPATCH_LEVEL) diff --git a/idd/LGAudio/Adapter.h b/idd/LGAudio/Adapter.h index 7174ff34..bca4820a 100644 --- a/idd/LGAudio/Adapter.h +++ b/idd/LGAudio/Adapter.h @@ -24,6 +24,15 @@ #include "Common.h" #include "RingBuffer.h" +struct AudioStreamSnapshot +{ + KSSTATE state; + bool formatValid; + UCHAR reserved[3]; + ULONGLONG generation; + FormatFingerprint format; +}; + class CAudioAdapter final : public IUnknown { public: @@ -33,9 +42,23 @@ public: NTSTATUS Initialize(); _IRQL_requires_max_(DISPATCH_LEVEL) - NTSTATUS ConfigureRender( + NTSTATUS ConfigureStream( + _In_ EndpointKind kind, _In_ const FormatFingerprint &format); + _IRQL_requires_max_(DISPATCH_LEVEL) + void SetStreamState( + _In_ EndpointKind kind, + _In_ KSSTATE state); + + _IRQL_requires_max_(DISPATCH_LEVEL) + void GetStreamSnapshot( + _In_ EndpointKind kind, + _Out_ AudioStreamSnapshot *snapshot); + + _IRQL_requires_max_(DISPATCH_LEVEL) + void SetBrokerActive(_In_ bool active); + _IRQL_requires_max_(DISPATCH_LEVEL) void ResetCapture(); @@ -55,6 +78,24 @@ public: _In_ const FormatFingerprint &format, _In_ ULONGLONG bytes); + _IRQL_requires_max_(DISPATCH_LEVEL) + NTSTATUS ReadRenderFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _Out_writes_bytes_(capacity) BYTE *data, + _In_ ULONG capacity, + _Out_ PULONG bytesRead, + _Out_ bool *discontinuity); + + _IRQL_requires_max_(DISPATCH_LEVEL) + NTSTATUS WriteCaptureFrames( + _In_ const FormatFingerprint &format, + _In_ ULONGLONG streamGeneration, + _In_ bool discontinuity, + _In_reads_bytes_(bytes) const BYTE *data, + _In_ ULONG bytes, + _Out_ PULONG droppedBytes); + _IRQL_requires_max_(DISPATCH_LEVEL) AudioClockSnapshot GetClockSnapshot() const; @@ -66,8 +107,15 @@ public: private: volatile LONG m_refCount; + volatile LONG m_brokerActive; + volatile LONG m_renderDiscontinuity; + KSPIN_LOCK m_transportLock; + KSPIN_LOCK m_streamStateLock; + AudioStreamSnapshot m_renderState; + AudioStreamSnapshot m_captureState; CAudioClock m_clock; - CAudioRingBuffer m_ring; + CAudioRingBuffer m_renderRing; + CAudioRingBuffer m_captureRing; }; NTSTATUS StartAudioDevice( diff --git a/idd/LGAudio/BrokerDevice.cpp b/idd/LGAudio/BrokerDevice.cpp index 3ef147c3..f660755e 100644 --- a/idd/LGAudio/BrokerDevice.cpp +++ b/idd/LGAudio/BrokerDevice.cpp @@ -31,6 +31,9 @@ extern "C" namespace { constexpr ULONG kBrokerFileSignature = 'fBGL'; + constexpr ULONG kBrokerReadPoolTag = 'qBGL'; + constexpr LONG kMaximumPendingReads = 32; + constexpr ULONG kReadServiceBatch = 8; struct BrokerFileContext { @@ -39,13 +42,33 @@ namespace UCHAR reserved[3]; ULONGLONG adapterGeneration; ULONGLONG bindingGeneration; + ULONGLONG captureStreamGeneration; + ULONGLONG nextCaptureSequence; + CAudioAdapter *adapter; }; struct BrokerControlExtension { KSPIN_LOCK stateLock; + FAST_MUTEX bindingLock; + FAST_MUTEX captureWriteLock; + EX_RUNDOWN_REF bindingRundown; + bool bindingRundownCompleted; PFILE_OBJECT owner; BrokerFileContext file; + IO_CSQ readCsq; + KSPIN_LOCK readQueueLock; + LIST_ENTRY readQueue; + KSPIN_LOCK pendingReadLock; + LONG pendingReadCount; + KEVENT readsDrainedEvent; + }; + + struct BrokerReadContext + { + ULONGLONG adapterGeneration; + ULONGLONG bindingGeneration; + ULONGLONG streamGeneration; }; struct AdapterSnapshot @@ -53,23 +76,36 @@ namespace bool present; ULONGLONG generation; ULONGLONG eventSequence; + AudioStreamSnapshot render; + AudioStreamSnapshot capture; + }; + + struct BindingSnapshot + { + CAudioAdapter *adapter; + ULONGLONG adapterGeneration; + ULONGLONG bindingGeneration; }; FAST_MUTEX g_controlLifecycleLock; FAST_MUTEX g_adapterLifecycleLock; KSPIN_LOCK g_adapterStateLock; EX_RUNDOWN_REF g_adapterRundown; + EX_RUNDOWN_REF g_controlServiceRundown; + KDPC g_readServiceDpc; PDRIVER_OBJECT g_driverObject; PDEVICE_OBJECT g_controlDevice; CAudioAdapter *g_adapter; PDEVICE_OBJECT g_adapterDevice; + CAudioAdapter *g_boundAdapter; volatile LONG g_initialized; volatile LONG g_retiring; volatile LONG g_pnpDeviceCount; bool g_linkCreated; bool g_rundownCompleted; + bool g_controlServiceRundownCompleted; volatile LONG64 g_adapterGeneration; volatile LONG64 g_eventSequence; @@ -78,6 +114,21 @@ namespace volatile LONG64 g_unbindCount; volatile LONG64 g_cleanupCount; volatile LONG64 g_rejectedRequestCount; + volatile LONG64 g_renderPacketCount; + volatile LONG64 g_renderByteCount; + volatile LONG64 g_renderDropBytes; + volatile LONG64 g_capturePacketCount; + volatile LONG64 g_captureByteCount; + volatile LONG64 g_captureDropBytes; + volatile LONG64 g_captureUnderrunBytes; + volatile LONG64 g_cancelledReadCount; + volatile LONG64 g_renderSequence; + + void ReadServiceDpc( + _In_ PKDPC dpc, + _In_opt_ PVOID deferredContext, + _In_opt_ PVOID systemArgument1, + _In_opt_ PVOID systemArgument2); BrokerControlExtension *GetControlExtension( _In_ PDEVICE_OBJECT deviceObject) @@ -97,6 +148,13 @@ namespace InterlockedIncrement64(&g_eventSequence); } + void AddCounter( + _Inout_ volatile LONG64 *counter, + _In_ ULONGLONG value) + { + InterlockedAdd64(counter, static_cast(value)); + } + PDEVICE_OBJECT LoadControlDevice() { return static_cast( @@ -113,6 +171,14 @@ namespace deviceObject)); } + CAudioAdapter *LoadBoundAdapter() + { + return static_cast( + InterlockedCompareExchangePointer( + reinterpret_cast(&g_boundAdapter), + nullptr, nullptr)); + } + NTSTATUS CompleteIrp( _Inout_ PIRP irp, _In_ NTSTATUS status, @@ -124,16 +190,196 @@ namespace return status; } + LONG AddPendingRead(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + const LONG count = ++extension->pendingReadCount; + if (count == 1) + KeClearEvent(&extension->readsDrainedEvent); + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + return count; + } + + void ReleasePendingRead(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + NT_ASSERT(extension->pendingReadCount > 0); + const LONG count = --extension->pendingReadCount; + if (count == 0) + KeSetEvent(&extension->readsDrainedEvent, IO_NO_INCREMENT, FALSE); + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + } + + bool HasPendingReads(_In_ BrokerControlExtension *extension) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->pendingReadLock, &oldIrql); + const bool pending = extension->pendingReadCount != 0; + KeReleaseSpinLock(&extension->pendingReadLock, oldIrql); + return pending; + } + + BrokerControlExtension *GetCsqExtension(_In_ PIO_CSQ csq) + { + return CONTAINING_RECORD(csq, BrokerControlExtension, readCsq); + } + + void CsqInsertIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + InsertTailList(&extension->readQueue, + &irp->Tail.Overlay.ListEntry); + } + + void CsqRemoveIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + UNREFERENCED_PARAMETER(csq); + RemoveEntryList(&irp->Tail.Overlay.ListEntry); + } + + PIRP CsqPeekNextIrp( + _In_ PIO_CSQ csq, + _In_opt_ PIRP irp, + _In_opt_ PVOID peekContext) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + PLIST_ENTRY entry = irp ? + irp->Tail.Overlay.ListEntry.Flink : + extension->readQueue.Flink; + + while (entry != &extension->readQueue) + { + PIRP candidate = CONTAINING_RECORD(entry, IRP, + Tail.Overlay.ListEntry); + if (!peekContext || + IoGetCurrentIrpStackLocation(candidate)->FileObject == + peekContext) + { + return candidate; + } + entry = entry->Flink; + } + return nullptr; + } + + void CsqAcquireLock(_In_ PIO_CSQ csq, _Out_ PKIRQL irql) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + KeAcquireSpinLock(&extension->readQueueLock, irql); + } + + void CsqReleaseLock(_In_ PIO_CSQ csq, _In_ KIRQL irql) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + KeReleaseSpinLock(&extension->readQueueLock, irql); + } + + void CsqCompleteCanceledIrp(_In_ PIO_CSQ csq, _In_ PIRP irp) + { + BrokerControlExtension *extension = GetCsqExtension(csq); + auto *context = static_cast( + irp->Tail.Overlay.DriverContext[0]); + irp->Tail.Overlay.DriverContext[0] = nullptr; + if (context) + ExFreePoolWithTag(context, kBrokerReadPoolTag); + InterlockedIncrement64(&g_cancelledReadCount); + CompleteIrp(irp, STATUS_CANCELLED); + ReleasePendingRead(extension); + } + + void SetReadContext( + _Inout_ PIRP irp, + _In_ BrokerReadContext *context) + { + irp->Tail.Overlay.DriverContext[0] = context; + } + + BrokerReadContext *GetReadContext(_In_ PIRP irp) + { + return static_cast( + irp->Tail.Overlay.DriverContext[0]); + } + + void FreeReadContext(_Inout_ PIRP irp) + { + BrokerReadContext *context = GetReadContext(irp); + irp->Tail.Overlay.DriverContext[0] = nullptr; + if (context) + ExFreePoolWithTag(context, kBrokerReadPoolTag); + } + + void CancelQueuedReads( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _In_ NTSTATUS status) + { + PIRP irp; + while ((irp = IoCsqRemoveNextIrp(&extension->readCsq, + fileObject)) != nullptr) + { + FreeReadContext(irp); + InterlockedIncrement64(&g_cancelledReadCount); + CompleteIrp(irp, status); + ReleasePendingRead(extension); + } + } + + void DrainQueuedReads( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _In_ NTSTATUS status) + { + CancelQueuedReads(extension, fileObject, status); + while (HasPendingReads(extension)) + { + KeWaitForSingleObject(&extension->readsDrainedEvent, + Executive, KernelMode, FALSE, nullptr); + } + } + + void QueueReadService() + { + if (!ExAcquireRundownProtection(&g_controlServiceRundown)) + return; + + if (LoadControlDevice() && + InterlockedCompareExchange(&g_retiring, 0, 0) == 0) + { + KeInsertQueueDpc(&g_readServiceDpc, nullptr, nullptr); + } + ExReleaseRundownProtection(&g_controlServiceRundown); + } + AdapterSnapshot GetAdapterSnapshot() { AdapterSnapshot snapshot{}; + if (!ExAcquireRundownProtection(&g_adapterRundown)) + { + snapshot.generation = ReadCounter(&g_adapterGeneration); + snapshot.eventSequence = ReadCounter(&g_eventSequence); + return snapshot; + } + + CAudioAdapter *adapter; KIRQL oldIrql; KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); - snapshot.present = g_adapter != nullptr; + adapter = g_adapter; + snapshot.present = adapter != nullptr; snapshot.generation = ReadCounter(&g_adapterGeneration); snapshot.eventSequence = ReadCounter(&g_eventSequence); KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + + if (adapter) + { + adapter->GetStreamSnapshot(EndpointKind::Render, + &snapshot.render); + adapter->GetStreamSnapshot(EndpointKind::Capture, + &snapshot.capture); + } + ExReleaseRundownProtection(&g_adapterRundown); return snapshot; } @@ -146,16 +392,83 @@ namespace extension->file.signature == kBrokerFileSignature; } - bool UnbindFileLocked(_In_ BrokerControlExtension *extension) + NTSTATUS AcquireBinding( + _In_ BrokerControlExtension *extension, + _In_opt_ PFILE_OBJECT fileObject, + _Out_ BindingSnapshot *snapshot) + { + RtlZeroMemory(snapshot, sizeof(*snapshot)); + if (!ExAcquireRundownProtection(&extension->bindingRundown)) + return STATUS_DEVICE_NOT_READY; + + NTSTATUS status = STATUS_SUCCESS; + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!IsOwnedFile(extension, fileObject)) + status = STATUS_INVALID_HANDLE; + else if (!extension->file.bound || !extension->file.adapter) + status = STATUS_DEVICE_NOT_READY; + else + { + snapshot->adapter = extension->file.adapter; + snapshot->adapterGeneration = + extension->file.adapterGeneration; + snapshot->bindingGeneration = + extension->file.bindingGeneration; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (!NT_SUCCESS(status)) + ExReleaseRundownProtection(&extension->bindingRundown); + return status; + } + + bool IsCurrentAdapter(_In_ const BindingSnapshot &binding) + { + bool current; + KIRQL oldIrql; + KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); + current = g_adapter == binding.adapter && + ReadCounter(&g_adapterGeneration) == + binding.adapterGeneration; + KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + return current; + } + + CAudioAdapter *UnbindFileLocked( + _In_ BrokerControlExtension *extension) { if (!extension->file.bound) - return false; + return nullptr; + CAudioAdapter *adapter = extension->file.adapter; extension->file.bound = false; extension->file.adapterGeneration = 0; extension->file.bindingGeneration = 0; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = nullptr; + InterlockedCompareExchangePointer( + reinterpret_cast(&g_boundAdapter), + nullptr, adapter); InterlockedIncrement64(&g_bindingGeneration); - return true; + return adapter; + } + + void FinishUnbind( + _In_ BrokerControlExtension *extension, + _In_ CAudioAdapter *adapter) + { + if (!adapter) + return; + + adapter->SetBrokerActive(false); + ExWaitForRundownProtectionRelease(&extension->bindingRundown); + extension->bindingRundownCompleted = true; + adapter->Release(); + + InterlockedIncrement64(&g_unbindCount); + IncrementEventSequence(); } _IRQL_requires_max_(PASSIVE_LEVEL) @@ -171,19 +484,17 @@ namespace BrokerControlExtension *extension = GetControlExtension(controlDevice); - bool unbound; + CAudioAdapter *adapter; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); - unbound = UnbindFileLocked(extension); + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); + FinishUnbind(extension, adapter); + DrainQueuedReads(extension, nullptr, STATUS_DEVICE_NOT_CONNECTED); + ExReleaseFastMutex(&extension->bindingLock); ExReleaseFastMutex(&g_controlLifecycleLock); - - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } } void DeleteSymbolicLinkLocked() @@ -197,6 +508,27 @@ namespace g_linkCreated = false; } + void StartControlServiceLocked() + { + if (!g_controlServiceRundownCompleted) + return; + + KeRemoveQueueDpc(&g_readServiceDpc); + ExReInitializeRundownProtection(&g_controlServiceRundown); + g_controlServiceRundownCompleted = false; + } + + void StopControlServiceLocked() + { + if (g_controlServiceRundownCompleted) + return; + + KeRemoveQueueDpc(&g_readServiceDpc); + ExWaitForRundownProtectionRelease(&g_controlServiceRundown); + KeRemoveQueueDpc(&g_readServiceDpc); + g_controlServiceRundownCompleted = true; + } + NTSTATUS CreateControlDeviceLocked() { if (LoadControlDevice()) @@ -229,9 +561,27 @@ namespace GetControlExtension(deviceObject); RtlZeroMemory(extension, sizeof(*extension)); KeInitializeSpinLock(&extension->stateLock); + ExInitializeFastMutex(&extension->bindingLock); + ExInitializeFastMutex(&extension->captureWriteLock); + ExInitializeRundownProtection(&extension->bindingRundown); + KeInitializeSpinLock(&extension->readQueueLock); + InitializeListHead(&extension->readQueue); + KeInitializeSpinLock(&extension->pendingReadLock); + KeInitializeEvent(&extension->readsDrainedEvent, + NotificationEvent, TRUE); + + status = IoCsqInitialize(&extension->readCsq, + CsqInsertIrp, CsqRemoveIrp, CsqPeekNextIrp, + CsqAcquireLock, CsqReleaseLock, CsqCompleteCanceledIrp); + if (!NT_SUCCESS(status)) + { + IoDeleteDevice(deviceObject); + return status; + } deviceObject->Flags |= DO_DIRECT_IO; + StartControlServiceLocked(); InterlockedExchange(&g_retiring, 0); const PDEVICE_OBJECT previous = static_cast( InterlockedCompareExchangePointer( @@ -285,7 +635,7 @@ namespace KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (InterlockedCompareExchange(&g_retiring, 0, 0) != 0 && - !extension->owner) + !extension->owner && !HasPendingReads(extension)) { const PDEVICE_OBJECT removed = ExchangeControlDevice(nullptr); NT_ASSERT(removed == controlDevice); @@ -325,6 +675,9 @@ namespace extension->file.bound = false; extension->file.adapterGeneration = 0; extension->file.bindingGeneration = 0; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = nullptr; fileObject->FsContext = &extension->file; } KeReleaseSpinLock(&extension->stateLock, oldIrql); @@ -341,20 +694,22 @@ namespace PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); BrokerControlExtension *extension = GetControlExtension(deviceObject); - bool unbound = false; + CAudioAdapter *adapter = nullptr; + bool owned; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); - if (IsOwnedFile(extension, stack->FileObject)) - unbound = UnbindFileLocked(extension); + owned = IsOwnedFile(extension, stack->FileObject); + if (owned) + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); + FinishUnbind(extension, adapter); + if (owned) + DrainQueuedReads(extension, stack->FileObject, STATUS_CANCELLED); + ExReleaseFastMutex(&extension->bindingLock); InterlockedIncrement64(&g_cleanupCount); - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } return CompleteIrp(irp, STATUS_SUCCESS); } @@ -366,25 +721,31 @@ namespace PFILE_OBJECT fileObject = stack->FileObject; BrokerControlExtension *extension = GetControlExtension(deviceObject); - bool unbound = false; + CAudioAdapter *adapter = nullptr; + bool owned; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + owned = IsOwnedFile(extension, fileObject); + if (owned) + adapter = UnbindFileLocked(extension); + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + FinishUnbind(extension, adapter); + if (owned) + DrainQueuedReads(extension, fileObject, STATUS_CANCELLED); + KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (IsOwnedFile(extension, fileObject)) { - unbound = UnbindFileLocked(extension); extension->owner = nullptr; extension->file.signature = 0; if (fileObject) fileObject->FsContext = nullptr; } KeReleaseSpinLock(&extension->stateLock, oldIrql); - - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } + ExReleaseFastMutex(&extension->bindingLock); const NTSTATUS status = CompleteIrp(irp, STATUS_SUCCESS); @@ -413,6 +774,75 @@ namespace header->version == LGAUDIO_BROKER_ABI_VERSION; } + bool ValidatePacketFormat( + _In_ const LGAudioBrokerFormat &format, + _Out_opt_ FormatFingerprint *fingerprint) + { + if (!format.sampleRate || + format.sampleRate > kMaximumSampleRate || + !format.channels || format.channels > 8 || + (format.containerBits != 16 && format.containerBits != 24) || + format.validBits != format.containerBits || + format.blockAlign != + format.channels * (format.containerBits / 8)) + { + return false; + } + + if (fingerprint) + { + fingerprint->sampleRate = format.sampleRate; + fingerprint->channelMask = format.channelMask; + fingerprint->channels = format.channels; + fingerprint->containerBits = format.containerBits; + fingerprint->validBits = format.validBits; + fingerprint->blockAlign = format.blockAlign; + } + return true; + } + + void CopyBrokerFormat( + _In_ const FormatFingerprint &source, + _Out_ LGAudioBrokerFormat *destination) + { + destination->sampleRate = source.sampleRate; + destination->channelMask = source.channelMask; + destination->channels = source.channels; + destination->containerBits = source.containerBits; + destination->validBits = source.validBits; + destination->blockAlign = source.blockAlign; + } + + UINT32 GetBrokerStreamState(_In_ KSSTATE state) + { + switch (state) + { + case KSSTATE_STOP: + return LGAUDIO_BROKER_STREAM_STOPPED; + case KSSTATE_ACQUIRE: + return LGAUDIO_BROKER_STREAM_ACQUIRED; + case KSSTATE_PAUSE: + return LGAUDIO_BROKER_STREAM_PAUSED; + case KSSTATE_RUN: + return LGAUDIO_BROKER_STREAM_RUNNING; + default: + return LGAUDIO_BROKER_STREAM_UNAVAILABLE; + } + } + + void FillBrokerStreamState( + _In_ const AudioStreamSnapshot &source, + _Out_ LGAudioBrokerStreamState *destination) + { + destination->generation = source.generation; + destination->state = GetBrokerStreamState(source.state); + if (source.formatValid) + { + destination->flags |= LGAUDIO_BROKER_STREAM_FORMAT_VALID; + CopyBrokerFormat(source.format, &destination->format); + } + } + NTSTATUS QueryCaps( _Inout_ PIRP irp, _In_ ULONG outputLength) @@ -428,7 +858,11 @@ namespace caps->version = LGAUDIO_BROKER_ABI_VERSION; caps->minimumVersion = LGAUDIO_BROKER_ABI_VERSION; caps->maximumVersion = LGAUDIO_BROKER_ABI_VERSION; - caps->capabilities = LGAUDIO_BROKER_CAP_CONTROL; + caps->capabilities = LGAUDIO_BROKER_CAP_CONTROL | + LGAUDIO_BROKER_CAP_RENDER_READ | + LGAUDIO_BROKER_CAP_CAPTURE_WRITE; + caps->maximumRenderTransfer = LGAUDIO_BROKER_MAX_PAYLOAD_BYTES; + caps->maximumCaptureTransfer = LGAUDIO_BROKER_MAX_PAYLOAD_BYTES; return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*caps)); } @@ -452,7 +886,8 @@ namespace if (IsOwnedFile(extension, fileObject)) { bound = extension->file.bound && snapshot.present && - extension->file.adapterGeneration == snapshot.generation; + extension->file.adapterGeneration == snapshot.generation && + extension->file.adapter == LoadBoundAdapter(); } KeReleaseSpinLock(&extension->stateLock, oldIrql); @@ -464,12 +899,22 @@ namespace state->adapterGeneration = snapshot.generation; state->eventSequence = snapshot.eventSequence; state->bindingGeneration = bindingGeneration; - state->render.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; - state->capture.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; if (snapshot.present) + { state->flags |= LGAUDIO_BROKER_STATE_ADAPTER_PRESENT; + FillBrokerStreamState(snapshot.render, &state->render); + FillBrokerStreamState(snapshot.capture, &state->capture); + } + else + { + state->render.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; + state->capture.state = LGAUDIO_BROKER_STREAM_UNAVAILABLE; + } if (bound) - state->flags |= LGAUDIO_BROKER_STATE_BOUND; + { + state->flags |= LGAUDIO_BROKER_STATE_BOUND | + LGAUDIO_BROKER_STATE_DATA_PLANE_ACTIVE; + } return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*state)); } @@ -506,22 +951,43 @@ namespace status = STATUS_RETRY; else { + ExAcquireFastMutex(&extension->bindingLock); + if (extension->bindingRundownCompleted) + { + ExReInitializeRundownProtection(&extension->bindingRundown); + extension->bindingRundownCompleted = false; + } + KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (!IsOwnedFile(extension, fileObject)) status = STATUS_INVALID_HANDLE; else if (extension->file.bound) status = STATUS_INVALID_DEVICE_STATE; - else + else if (LoadBoundAdapter()) + status = STATUS_DEVICE_BUSY; + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (NT_SUCCESS(status)) { const ULONGLONG bindingGeneration = static_cast( InterlockedIncrement64(&g_bindingGeneration)); + adapter->AddRef(); + adapter->SetBrokerActive(true); + InterlockedExchangePointer( + reinterpret_cast(&g_boundAdapter), adapter); + + KeAcquireSpinLock(&extension->stateLock, &oldIrql); extension->file.bound = true; extension->file.adapterGeneration = generation; extension->file.bindingGeneration = bindingGeneration; + extension->file.captureStreamGeneration = 0; + extension->file.nextCaptureSequence = 0; + extension->file.adapter = adapter; bind->bindingGeneration = bindingGeneration; + KeReleaseSpinLock(&extension->stateLock, oldIrql); } - KeReleaseSpinLock(&extension->stateLock, oldIrql); + ExReleaseFastMutex(&extension->bindingLock); } BrokerDeviceReleaseAdapter(); @@ -545,21 +1011,20 @@ namespace sizeof(LGAudioBrokerHeader))) return CompleteIrp(irp, STATUS_INVALID_PARAMETER); - bool unbound = false; + CAudioAdapter *adapter = nullptr; NTSTATUS status = STATUS_SUCCESS; + ExAcquireFastMutex(&extension->bindingLock); KIRQL oldIrql; KeAcquireSpinLock(&extension->stateLock, &oldIrql); if (!IsOwnedFile(extension, fileObject)) status = STATUS_INVALID_HANDLE; else - unbound = UnbindFileLocked(extension); + adapter = UnbindFileLocked(extension); KeReleaseSpinLock(&extension->stateLock, oldIrql); - if (unbound) - { - InterlockedIncrement64(&g_unbindCount); - IncrementEventSequence(); - } + FinishUnbind(extension, adapter); + DrainQueuedReads(extension, fileObject, STATUS_CANCELLED); + ExReleaseFastMutex(&extension->bindingLock); return CompleteIrp(irp, status); } @@ -584,9 +1049,401 @@ namespace stats->cleanupCount = ReadCounter(&g_cleanupCount); stats->rejectedRequestCount = ReadCounter(&g_rejectedRequestCount); + stats->renderPacketCount = ReadCounter(&g_renderPacketCount); + stats->renderByteCount = ReadCounter(&g_renderByteCount); + stats->renderDropBytes = ReadCounter(&g_renderDropBytes); + stats->capturePacketCount = ReadCounter(&g_capturePacketCount); + stats->captureByteCount = ReadCounter(&g_captureByteCount); + stats->captureDropBytes = ReadCounter(&g_captureDropBytes); + stats->captureUnderrunBytes = + ReadCounter(&g_captureUnderrunBytes); + stats->cancelledReadCount = ReadCounter(&g_cancelledReadCount); return CompleteIrp(irp, STATUS_SUCCESS, sizeof(*stats)); } + void *MapDirectBuffer( + _In_ PIRP irp, + _In_ ULONG requiredBytes) + { + if (!irp->MdlAddress || + MmGetMdlByteCount(irp->MdlAddress) < requiredBytes) + { + return nullptr; + } + + return MmGetSystemAddressForMdlSafe(irp->MdlAddress, + NormalPagePriority | MdlMappingNoExecute); + } + + void CompletePendingRead( + _In_ BrokerControlExtension *extension, + _Inout_ PIRP irp, + _In_ NTSTATUS status, + _In_ ULONG_PTR information = 0) + { + FreeReadContext(irp); + if (status == STATUS_CANCELLED || + status == STATUS_DEVICE_NOT_READY || + status == STATUS_DEVICE_NOT_CONNECTED || + status == STATUS_RETRY) + { + InterlockedIncrement64(&g_cancelledReadCount); + } + CompleteIrp(irp, status, information); + ReleasePendingRead(extension); + } + + NTSTATUS DispatchRead( + _In_ PDEVICE_OBJECT deviceObject, + _Inout_ PIRP irp) + { + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + const ULONG length = stack->Parameters.Read.Length; + if (length <= sizeof(LGAudioBrokerPacketHeader) || + length > sizeof(LGAudioBrokerPacketHeader) + + LGAUDIO_BROKER_MAX_PAYLOAD_BYTES || + !MapDirectBuffer(irp, length)) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + BrokerControlExtension *extension = + GetControlExtension(deviceObject); + BindingSnapshot binding{}; + NTSTATUS status = AcquireBinding(extension, stack->FileObject, + &binding); + if (!NT_SUCCESS(status)) + return CompleteIrp(irp, status); + + AudioStreamSnapshot render{}; + binding.adapter->GetStreamSnapshot(EndpointKind::Render, &render); + const ULONG payloadCapacity = + length - sizeof(LGAudioBrokerPacketHeader); + if (!IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + else if (!render.formatValid || render.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (!render.format.blockAlign || + payloadCapacity < render.format.blockAlign) + status = STATUS_INVALID_PARAMETER; + + if (!NT_SUCCESS(status)) + { + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, status); + } + +#pragma warning(push) +#pragma warning(disable:4996) // Required for pre-2004 Windows 10 releases. + auto *context = static_cast( + ExAllocatePoolWithTag(NonPagedPoolNx, sizeof(BrokerReadContext), + kBrokerReadPoolTag)); +#pragma warning(pop) + if (!context) + { + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, STATUS_INSUFFICIENT_RESOURCES); + } + + context->adapterGeneration = binding.adapterGeneration; + context->bindingGeneration = binding.bindingGeneration; + context->streamGeneration = render.generation; + SetReadContext(irp, context); + + const LONG pending = AddPendingRead(extension); + if (pending > kMaximumPendingReads) + { + ReleasePendingRead(extension); + FreeReadContext(irp); + ExReleaseRundownProtection(&extension->bindingRundown); + return CompleteIrp(irp, STATUS_DEVICE_BUSY); + } + + IoMarkIrpPending(irp); + IoCsqInsertIrp(&extension->readCsq, irp, nullptr); + ExReleaseRundownProtection(&extension->bindingRundown); + QueueReadService(); + return STATUS_PENDING; + } + + bool ServiceReadIrp( + _In_ BrokerControlExtension *extension, + _Inout_ PIRP irp) + { + BrokerReadContext *context = GetReadContext(irp); + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + if (!context || + !ExAcquireRundownProtection(&extension->bindingRundown)) + { + CompletePendingRead(extension, irp, STATUS_DEVICE_NOT_READY); + return true; + } + + BindingSnapshot binding{}; + NTSTATUS status = STATUS_SUCCESS; + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!IsOwnedFile(extension, stack->FileObject) || + !extension->file.bound || !extension->file.adapter) + { + status = STATUS_DEVICE_NOT_READY; + } + else + { + binding.adapter = extension->file.adapter; + binding.adapterGeneration = extension->file.adapterGeneration; + binding.bindingGeneration = extension->file.bindingGeneration; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + + if (NT_SUCCESS(status) && + (binding.adapterGeneration != context->adapterGeneration || + binding.bindingGeneration != context->bindingGeneration)) + { + status = STATUS_RETRY; + } + if (NT_SUCCESS(status) && !IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + + AudioStreamSnapshot render{}; + if (NT_SUCCESS(status)) + { + binding.adapter->GetStreamSnapshot(EndpointKind::Render, &render); + if (!render.formatValid || render.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (render.generation != context->streamGeneration) + status = STATUS_RETRY; + } + + const ULONG length = stack->Parameters.Read.Length; + auto *buffer = static_cast(MapDirectBuffer(irp, length)); + if (NT_SUCCESS(status) && !buffer) + status = STATUS_INSUFFICIENT_RESOURCES; + + ULONG bytesRead = 0; + bool discontinuity = false; + if (NT_SUCCESS(status)) + { + const ULONG capacity = + length - sizeof(LGAudioBrokerPacketHeader); + BYTE *payload = buffer + sizeof(LGAudioBrokerPacketHeader); + status = binding.adapter->ReadRenderFrames(render.format, + render.generation, payload, capacity, &bytesRead, + &discontinuity); + } + + if (NT_SUCCESS(status) && !bytesRead) + { + IoCsqInsertIrp(&extension->readCsq, irp, nullptr); + ExReleaseRundownProtection(&extension->bindingRundown); + return false; + } + + if (NT_SUCCESS(status)) + { + LGAudioBrokerPacketHeader packet{}; + packet.size = sizeof(packet); + packet.version = LGAUDIO_BROKER_ABI_VERSION; + packet.flags = discontinuity ? + LGAUDIO_BROKER_PACKET_DISCONTINUITY : 0; + packet.direction = LGAUDIO_BROKER_PACKET_RENDER; + packet.adapterGeneration = binding.adapterGeneration; + packet.bindingGeneration = binding.bindingGeneration; + packet.streamGeneration = render.generation; + packet.sequence = static_cast( + InterlockedIncrement64(&g_renderSequence) - 1); + CopyBrokerFormat(render.format, &packet.format); + packet.frameCount = bytesRead / render.format.blockAlign; + packet.payloadBytes = bytesRead; + RtlCopyMemory(buffer, &packet, sizeof(packet)); + + InterlockedIncrement64(&g_renderPacketCount); + AddCounter(&g_renderByteCount, bytesRead); + } + + const ULONG_PTR information = NT_SUCCESS(status) ? + sizeof(LGAudioBrokerPacketHeader) + bytesRead : 0; + CompletePendingRead(extension, irp, status, information); + ExReleaseRundownProtection(&extension->bindingRundown); + return true; + } + + void ReadServiceDpc( + _In_ PKDPC dpc, + _In_opt_ PVOID deferredContext, + _In_opt_ PVOID systemArgument1, + _In_opt_ PVOID systemArgument2) + { + UNREFERENCED_PARAMETER(dpc); + UNREFERENCED_PARAMETER(deferredContext); + UNREFERENCED_PARAMETER(systemArgument1); + UNREFERENCED_PARAMETER(systemArgument2); + + if (!ExAcquireRundownProtection(&g_controlServiceRundown)) + return; + + PDEVICE_OBJECT controlDevice = LoadControlDevice(); + ULONG serviced = 0; + if (controlDevice) + { + BrokerControlExtension *extension = + GetControlExtension(controlDevice); + while (serviced < kReadServiceBatch) + { + PIRP irp = IoCsqRemoveNextIrp(&extension->readCsq, nullptr); + if (!irp) + break; + if (!ServiceReadIrp(extension, irp)) + break; + ++serviced; + } + + if (serviced == kReadServiceBatch && + HasPendingReads(extension)) + { + KeInsertQueueDpc(&g_readServiceDpc, nullptr, nullptr); + } + } + + ExReleaseRundownProtection(&g_controlServiceRundown); + } + + NTSTATUS DispatchWrite( + _In_ PDEVICE_OBJECT deviceObject, + _Inout_ PIRP irp) + { + PIO_STACK_LOCATION stack = IoGetCurrentIrpStackLocation(irp); + const ULONG length = stack->Parameters.Write.Length; + if (length <= sizeof(LGAudioBrokerPacketHeader) || + length > sizeof(LGAudioBrokerPacketHeader) + + LGAUDIO_BROKER_MAX_PAYLOAD_BYTES) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + const auto *mapped = static_cast( + MapDirectBuffer(irp, length)); + if (!mapped) + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + + LGAudioBrokerPacketHeader packet{}; + RtlCopyMemory(&packet, mapped, sizeof(packet)); + FormatFingerprint format{}; + const UINT32 allowedFlags = + LGAUDIO_BROKER_PACKET_DISCONTINUITY; + if (packet.size != sizeof(packet) || + packet.version != LGAUDIO_BROKER_ABI_VERSION || + (packet.flags & ~allowedFlags) != 0 || + packet.direction != LGAUDIO_BROKER_PACKET_CAPTURE || + packet.reserved[0] || packet.reserved[1] || + !packet.payloadBytes || + packet.payloadBytes > LGAUDIO_BROKER_MAX_PAYLOAD_BYTES || + packet.payloadBytes != length - sizeof(packet) || + !ValidatePacketFormat(packet.format, &format) || + !packet.frameCount || + (packet.payloadBytes % format.blockAlign) != 0 || + packet.frameCount != packet.payloadBytes / format.blockAlign) + { + return CompleteIrp(irp, STATUS_INVALID_PARAMETER); + } + + BrokerControlExtension *extension = + GetControlExtension(deviceObject); + ExAcquireFastMutex(&extension->captureWriteLock); + + BindingSnapshot binding{}; + NTSTATUS status = AcquireBinding(extension, stack->FileObject, + &binding); + if (NT_SUCCESS(status) && + (packet.adapterGeneration != binding.adapterGeneration || + packet.bindingGeneration != binding.bindingGeneration)) + { + status = STATUS_RETRY; + } + if (NT_SUCCESS(status) && !IsCurrentAdapter(binding)) + status = STATUS_DEVICE_NOT_CONNECTED; + + AudioStreamSnapshot capture{}; + if (NT_SUCCESS(status)) + { + binding.adapter->GetStreamSnapshot(EndpointKind::Capture, + &capture); + if (!capture.formatValid || capture.state != KSSTATE_RUN) + status = STATUS_DEVICE_NOT_READY; + else if (packet.streamGeneration != capture.generation || + !EqualFormatFingerprint(format, capture.format)) + status = STATUS_RETRY; + } + + if (NT_SUCCESS(status)) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!extension->file.bound || + extension->file.adapter != binding.adapter || + extension->file.bindingGeneration != + binding.bindingGeneration) + { + status = STATUS_DEVICE_NOT_READY; + } + else if (extension->file.captureStreamGeneration != + capture.generation) + { + if (packet.sequence != 0) + status = STATUS_RETRY; + } + else if (packet.sequence != + extension->file.nextCaptureSequence) + { + status = STATUS_RETRY; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + } + + ULONG droppedBytes = 0; + if (NT_SUCCESS(status)) + { + const BYTE *payload = + mapped + sizeof(LGAudioBrokerPacketHeader); + status = binding.adapter->WriteCaptureFrames(format, + capture.generation, + (packet.flags & LGAUDIO_BROKER_PACKET_DISCONTINUITY) != 0, + payload, packet.payloadBytes, &droppedBytes); + } + + if (NT_SUCCESS(status)) + { + KIRQL oldIrql; + KeAcquireSpinLock(&extension->stateLock, &oldIrql); + if (!extension->file.bound || + extension->file.bindingGeneration != + binding.bindingGeneration) + { + status = STATUS_DEVICE_NOT_READY; + } + else + { + extension->file.captureStreamGeneration = capture.generation; + extension->file.nextCaptureSequence = packet.sequence + 1; + } + KeReleaseSpinLock(&extension->stateLock, oldIrql); + } + + if (NT_SUCCESS(status)) + { + InterlockedIncrement64(&g_capturePacketCount); + AddCounter(&g_captureByteCount, packet.payloadBytes); + AddCounter(&g_captureDropBytes, droppedBytes); + } + + ExReleaseFastMutex(&extension->captureWriteLock); + const NTSTATUS completedStatus = CompleteIrp(irp, status, + NT_SUCCESS(status) ? length : 0); + if (binding.adapter) + ExReleaseRundownProtection(&extension->bindingRundown); + return completedStatus; + } + NTSTATUS DispatchDeviceControl( _In_ PDEVICE_OBJECT deviceObject, _Inout_ PIRP irp) @@ -659,6 +1516,8 @@ NTSTATUS BrokerDeviceInitialize(_In_ PDRIVER_OBJECT driverObject) ExInitializeFastMutex(&g_adapterLifecycleLock); KeInitializeSpinLock(&g_adapterStateLock); ExInitializeRundownProtection(&g_adapterRundown); + ExInitializeRundownProtection(&g_controlServiceRundown); + KeInitializeDpc(&g_readServiceDpc, ReadServiceDpc, nullptr); g_driverObject = driverObject; ExAcquireFastMutex(&g_controlLifecycleLock); @@ -699,6 +1558,7 @@ void BrokerDeviceShutdown() ExAcquireFastMutex(&g_controlLifecycleLock); SetControlRetiringLocked(true); DeleteSymbolicLinkLocked(); + StopControlServiceLocked(); deleteDevice = TakeRetiredControlDeviceLocked(); NT_ASSERT(deleteDevice || !LoadControlDevice()); ExReleaseFastMutex(&g_controlLifecycleLock); @@ -741,9 +1601,20 @@ NTSTATUS BrokerDeviceDispatch( return CompleteIrp(irp, STATUS_SUCCESS); case IRP_MJ_READ: + { + const NTSTATUS status = DispatchRead(deviceObject, irp); + if (!NT_SUCCESS(status)) + InterlockedIncrement64(&g_rejectedRequestCount); + return status; + } + case IRP_MJ_WRITE: - InterlockedIncrement64(&g_rejectedRequestCount); - return CompleteIrp(irp, STATUS_NOT_SUPPORTED); + { + const NTSTATUS status = DispatchWrite(deviceObject, irp); + if (!NT_SUCCESS(status)) + InterlockedIncrement64(&g_rejectedRequestCount); + return status; + } default: InterlockedIncrement64(&g_rejectedRequestCount); @@ -778,6 +1649,7 @@ void BrokerDeviceNotifyPnpDeviceAdded() if (NT_SUCCESS(status)) { g_linkCreated = true; + StartControlServiceLocked(); SetControlRetiringLocked(false); } else @@ -803,6 +1675,7 @@ void BrokerDeviceNotifyPnpDeviceRemoved() ExAcquireFastMutex(&g_controlLifecycleLock); SetControlRetiringLocked(true); DeleteSymbolicLinkLocked(); + StopControlServiceLocked(); deleteDevice = TakeRetiredControlDeviceLocked(); ExReleaseFastMutex(&g_controlLifecycleLock); @@ -923,3 +1796,44 @@ void BrokerDeviceReleaseAdapter() { ExReleaseRundownProtection(&g_adapterRundown); } + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyStreamChanged( + _In_ CAudioAdapter *adapter, + _In_ EndpointKind kind) +{ + bool current; + KIRQL oldIrql; + KeAcquireSpinLock(&g_adapterStateLock, &oldIrql); + current = adapter && g_adapter == adapter; + KeReleaseSpinLock(&g_adapterStateLock, oldIrql); + if (!current) + return; + + IncrementEventSequence(); + if (kind == EndpointKind::Render) + QueueReadService(); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyRenderAvailable( + _In_ CAudioAdapter *adapter, + _In_ ULONG bytes, + _In_ ULONG droppedBytes) +{ + UNREFERENCED_PARAMETER(bytes); + if (!adapter || LoadBoundAdapter() != adapter) + return; + + AddCounter(&g_renderDropBytes, droppedBytes); + QueueReadService(); +} + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceRecordCaptureUnderrun( + _In_ CAudioAdapter *adapter, + _In_ ULONGLONG bytes) +{ + if (adapter && LoadBoundAdapter() == adapter) + AddCounter(&g_captureUnderrunBytes, bytes); +} diff --git a/idd/LGAudio/BrokerDevice.h b/idd/LGAudio/BrokerDevice.h index f78aaaa3..43a3c0d2 100644 --- a/idd/LGAudio/BrokerDevice.h +++ b/idd/LGAudio/BrokerDevice.h @@ -21,6 +21,7 @@ #pragma once #include "Common.h" +#include "Formats.h" class CAudioAdapter; @@ -75,3 +76,19 @@ CAudioAdapter *BrokerDeviceAcquireAdapter( _IRQL_requires_max_(DISPATCH_LEVEL) void BrokerDeviceReleaseAdapter(); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyStreamChanged( + _In_ CAudioAdapter *adapter, + _In_ EndpointKind kind); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceNotifyRenderAvailable( + _In_ CAudioAdapter *adapter, + _In_ ULONG bytes, + _In_ ULONG droppedBytes); + +_IRQL_requires_max_(DISPATCH_LEVEL) +void BrokerDeviceRecordCaptureUnderrun( + _In_ CAudioAdapter *adapter, + _In_ ULONGLONG bytes); diff --git a/idd/LGAudio/MiniportWaveRTStream.cpp b/idd/LGAudio/MiniportWaveRTStream.cpp index c4ed955e..445c9c4c 100644 --- a/idd/LGAudio/MiniportWaveRTStream.cpp +++ b/idd/LGAudio/MiniportWaveRTStream.cpp @@ -114,8 +114,7 @@ NTSTATUS CMiniportWaveRTStream::Initialize( m_format = *format; m_fingerprint = fingerprint; - if (m_kind == EndpointKind::Render) - status = m_adapter->ConfigureRender(m_fingerprint); + status = m_adapter->ConfigureStream(m_kind, m_fingerprint); return status; } @@ -195,8 +194,7 @@ NTSTATUS CMiniportWaveRTStream::SetFormat( m_fingerprint = fingerprint; KeReleaseSpinLock(&m_positionLock, oldIrql); - if (m_kind == EndpointKind::Render) - status = m_adapter->ConfigureRender(m_fingerprint); + status = m_adapter->ConfigureStream(m_kind, m_fingerprint); return status; } @@ -244,6 +242,8 @@ NTSTATUS CMiniportWaveRTStream::SetState(_In_ KSSTATE state) if (resetCapture) m_adapter->ResetCapture(); + m_adapter->SetStreamState(m_kind, KSSTATE_RUN); + KeAcquireSpinLock(&m_positionLock, &oldIrql); if (m_state == KSSTATE_RUN) ArmTimerLocked(); @@ -280,6 +280,7 @@ NTSTATUS CMiniportWaveRTStream::SetState(_In_ KSSTATE state) KeReleaseSpinLock(&m_positionLock, oldIrql); StopTimer(); + m_adapter->SetStreamState(m_kind, state); return STATUS_SUCCESS; } @@ -901,6 +902,7 @@ void CMiniportWaveRTStream::AdvanceEventPacketsLocked( { m_eosNotificationSent = true; m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); m_pendingBytes = 0; m_frameRemainder = 0; break; @@ -944,6 +946,7 @@ void CMiniportWaveRTStream::TransferFramesLocked(_In_ ULONGLONG bytes) if (m_linearPosition >= m_eosLinearPosition) { m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); return; } @@ -997,7 +1000,10 @@ void CMiniportWaveRTStream::TransferFramesLocked(_In_ ULONGLONG bytes) if (m_kind == EndpointKind::Render && m_eosReceived && m_linearPosition >= m_eosLinearPosition) + { m_state = KSSTATE_PAUSE; + m_adapter->SetStreamState(m_kind, KSSTATE_PAUSE); + } } void CMiniportWaveRTStream::SignalNotificationsLocked() diff --git a/idd/LGCommon/LGAudioBroker.h b/idd/LGCommon/LGAudioBroker.h index 9dff30fc..1ab7f380 100644 --- a/idd/LGCommon/LGAudioBroker.h +++ b/idd/LGCommon/LGAudioBroker.h @@ -44,11 +44,12 @@ static const GUID LGAUDIO_BROKER_DEVICE_CLASS_GUID = { 0xb4, 0x90, 0x28, 0xc5, 0x30, 0xd2, 0x38, 0xcd } }; static constexpr UINT32 LGAUDIO_BROKER_ABI_VERSION = 1U; +// The transfer limits in LGAudioBrokerCaps exclude the packet header. +static constexpr UINT32 LGAUDIO_BROKER_MAX_PAYLOAD_BYTES = 64U * 1024U; enum LGAudioBrokerCapability : UINT32 { LGAUDIO_BROKER_CAP_CONTROL = 1U << 0, - // These remain clear until the corresponding data plane is implemented. LGAUDIO_BROKER_CAP_RENDER_READ = 1U << 1, LGAUDIO_BROKER_CAP_CAPTURE_WRITE = 1U << 2, LGAUDIO_BROKER_CAP_CLOCK_CONTROL = 1U << 3 @@ -70,6 +71,22 @@ enum LGAudioBrokerStreamStateValue : UINT32 LGAUDIO_BROKER_STREAM_RUNNING = 4 }; +enum LGAudioBrokerStreamFlag : UINT32 +{ + LGAUDIO_BROKER_STREAM_FORMAT_VALID = 1U << 0 +}; + +enum LGAudioBrokerPacketDirection : UINT32 +{ + LGAUDIO_BROKER_PACKET_RENDER = 1, + LGAUDIO_BROKER_PACKET_CAPTURE = 2 +}; + +enum LGAudioBrokerPacketFlag : UINT32 +{ + LGAUDIO_BROKER_PACKET_DISCONTINUITY = 1U << 0 +}; + static constexpr UINT32 IOCTL_LGAUDIO_BROKER_QUERY_CAPS = CTL_CODE(FILE_DEVICE_UNKNOWN, 0x900, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS); @@ -106,12 +123,33 @@ struct LGAudioBrokerFormat struct LGAudioBrokerStreamState { + // Changes on format configuration and each transition to STOP. UINT64 generation; UINT32 state; UINT32 flags; LGAudioBrokerFormat format; }; +struct LGAudioBrokerPacketHeader +{ + // READ returns this header followed by render payload. WRITE accepts this + // header followed by capture payload. size is always the header size. + UINT32 size; + UINT32 version; + UINT32 flags; + UINT32 direction; + UINT64 adapterGeneration; + UINT64 bindingGeneration; + UINT64 streamGeneration; + // Render sequence is monotonic for the driver lifetime. Capture starts at + // zero for each binding/stream-generation pair and must be contiguous. + UINT64 sequence; + LGAudioBrokerFormat format; + UINT32 frameCount; + UINT32 payloadBytes; + UINT32 reserved[2]; +}; + struct LGAudioBrokerCaps { UINT32 size; @@ -160,7 +198,15 @@ struct LGAudioBrokerStats UINT64 unbindCount; UINT64 cleanupCount; UINT64 rejectedRequestCount; - UINT64 reserved[2]; + UINT64 renderPacketCount; + UINT64 renderByteCount; + UINT64 renderDropBytes; + UINT64 capturePacketCount; + UINT64 captureByteCount; + UINT64 captureDropBytes; + UINT64 captureUnderrunBytes; + UINT64 cancelledReadCount; + UINT64 reserved[4]; }; #pragma pack(pop) @@ -171,11 +217,16 @@ static_assert(sizeof(LGAudioBrokerFormat) == 16, "LGAudio broker format layout changed"); static_assert(sizeof(LGAudioBrokerStreamState) == 32, "LGAudio broker stream-state layout changed"); +static_assert(sizeof(LGAudioBrokerPacketHeader) == 80, + "LGAudio broker packet layout changed"); static_assert(sizeof(LGAudioBrokerCaps) == 64, "LGAudio broker capabilities layout changed"); static_assert(sizeof(LGAudioBrokerState) == 104, "LGAudio broker state layout changed"); static_assert(sizeof(LGAudioBrokerBind) == 48, "LGAudio broker binding layout changed"); -static_assert(sizeof(LGAudioBrokerStats) == 80, +static_assert(sizeof(LGAudioBrokerStats) == 160, "LGAudio broker statistics layout changed"); +static_assert(LGAUDIO_BROKER_MAX_PAYLOAD_BYTES >= + (192000U / 100U) * 8U * 3U, + "LGAudio broker payload cannot hold 10 ms at the maximum format");