From 26cafbe3dfbeef984670917d3b4c4aab9d60fbb7 Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Mon, 17 Aug 2026 01:27:38 +1000 Subject: [PATCH] [idd] lgmp: stream staged copies to WC memory --- idd/LGCommon/LGCommon.vcxproj | 5 + idd/LGCommon/LGCommon.vcxproj.filters | 9 ++ idd/LGCommon/WCWrite.cpp | 146 ++++++++++++++++++ idd/LGCommon/WCWrite.h | 32 ++++ idd/LGCommon/WCWriteAVX2.cpp | 84 ++++++++++ .../transport/lgmp/CLGMPFrameTransport.cpp | 15 +- 6 files changed, 284 insertions(+), 7 deletions(-) create mode 100644 idd/LGCommon/WCWrite.cpp create mode 100644 idd/LGCommon/WCWrite.h create mode 100644 idd/LGCommon/WCWriteAVX2.cpp diff --git a/idd/LGCommon/LGCommon.vcxproj b/idd/LGCommon/LGCommon.vcxproj index 408e8b97..047d020b 100644 --- a/idd/LGCommon/LGCommon.vcxproj +++ b/idd/LGCommon/LGCommon.vcxproj @@ -74,6 +74,10 @@ AdvancedVectorExtensions2 + + + AdvancedVectorExtensions2 + @@ -90,6 +94,7 @@ + diff --git a/idd/LGCommon/LGCommon.vcxproj.filters b/idd/LGCommon/LGCommon.vcxproj.filters index 8bc23dc1..1a28d499 100644 --- a/idd/LGCommon/LGCommon.vcxproj.filters +++ b/idd/LGCommon/LGCommon.vcxproj.filters @@ -33,6 +33,12 @@ Source Files + + Source Files + + + Source Files + @@ -77,5 +83,8 @@ Header Files + + Header Files + diff --git a/idd/LGCommon/WCWrite.cpp b/idd/LGCommon/WCWrite.cpp new file mode 100644 index 00000000..1f97a8e0 --- /dev/null +++ b/idd/LGCommon/WCWrite.cpp @@ -0,0 +1,146 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "WCWrite.h" + +#include +#include +#include +#include +#include + +namespace WCWrite +{ + namespace Detail + { + void CopyAVX2(void * destination, const void * source, size_t size); + } + + namespace + { + using CopyFn = void (*)(void *, const void *, size_t); + static constexpr unsigned __int64 AVX_XSTATE_MASK = 0x6U; + + void CopyFallback(void * destination, const void * source, size_t size) + { + memcpy(destination, source, size); + } + +#if defined(_M_IX86) || defined(_M_X64) + __declspec(noinline) void CopySSE41( + void * destination, const void * source, size_t size) + { + uint8_t * dst = static_cast(destination); + const uint8_t * src = static_cast(source); + + const size_t prefix = + (16U - (reinterpret_cast(dst) & 15U)) & 15U; + if (prefix) + { + const size_t copy = prefix < size ? prefix : size; + memcpy(dst, src, copy); + dst += copy; + src += copy; + size -= copy; + } + + while (size >= 64U) + { + const __m128i v0 = _mm_loadu_si128( + reinterpret_cast(src + 0)); + const __m128i v1 = _mm_loadu_si128( + reinterpret_cast(src + 16)); + const __m128i v2 = _mm_loadu_si128( + reinterpret_cast(src + 32)); + const __m128i v3 = _mm_loadu_si128( + reinterpret_cast(src + 48)); + __m128i * output = reinterpret_cast<__m128i *>(dst); + _mm_stream_si128(output + 0, v0); + _mm_stream_si128(output + 1, v1); + _mm_stream_si128(output + 2, v2); + _mm_stream_si128(output + 3, v3); + dst += 64U; + src += 64U; + size -= 64U; + } + + while (size >= 16U) + { + const __m128i value = _mm_loadu_si128( + reinterpret_cast(src)); + _mm_stream_si128(reinterpret_cast<__m128i *>(dst), value); + dst += 16U; + src += 16U; + size -= 16U; + } + + if (size) + memcpy(dst, src, size); + } + + CopyFn SelectCopy() + { + int registers[4] = {}; + __cpuid(registers, 0); + const int maximumLeaf = registers[0]; + if (maximumLeaf < 1) + return &CopyFallback; + + __cpuidex(registers, 1, 0); + const bool sse41 = (registers[2] & (1 << 19)) != 0; + const bool avx = (registers[2] & (1 << 28)) != 0; + const bool osxsave = (registers[2] & (1 << 27)) != 0; + + if (maximumLeaf >= 7 && avx && osxsave && + (_xgetbv(0) & AVX_XSTATE_MASK) == AVX_XSTATE_MASK) + { + __cpuidex(registers, 7, 0); + if ((registers[1] & (1 << 5)) != 0) + return &Detail::CopyAVX2; + } + + return sse41 ? &CopySSE41 : &CopyFallback; + } +#else + CopyFn SelectCopy() + { + return &CopyFallback; + } +#endif + } + + void Copy(void * destination, const void * source, size_t size) + { + if (!size) + return; + + static const CopyFn copy = SelectCopy(); + copy(destination, source, size); + } + + void Flush() + { +#if defined(_M_IX86) || defined(_M_X64) + _mm_sfence(); +#else + std::atomic_thread_fence(std::memory_order_release); +#endif + } +} diff --git a/idd/LGCommon/WCWrite.h b/idd/LGCommon/WCWrite.h new file mode 100644 index 00000000..5d22c4a7 --- /dev/null +++ b/idd/LGCommon/WCWrite.h @@ -0,0 +1,32 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include + +namespace WCWrite +{ + // Copy from normal cacheable memory into write-combined memory. The source + // and destination ranges must not overlap. Call Flush before publishing the + // written range to another processor or device. + void Copy(void * destination, const void * source, size_t size); + void Flush(); +} diff --git a/idd/LGCommon/WCWriteAVX2.cpp b/idd/LGCommon/WCWriteAVX2.cpp new file mode 100644 index 00000000..c3423516 --- /dev/null +++ b/idd/LGCommon/WCWriteAVX2.cpp @@ -0,0 +1,84 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "WCWrite.h" + +#include +#include +#include + +namespace WCWrite +{ + namespace Detail + { + __declspec(noinline) void CopyAVX2( + void * destination, const void * source, size_t size) + { + uint8_t * dst = static_cast(destination); + const uint8_t * src = static_cast(source); + + const size_t prefix = + (32U - (reinterpret_cast(dst) & 31U)) & 31U; + if (prefix) + { + const size_t copy = prefix < size ? prefix : size; + memcpy(dst, src, copy); + dst += copy; + src += copy; + size -= copy; + } + + while (size >= 128U) + { + const __m256i v0 = _mm256_loadu_si256( + reinterpret_cast(src + 0)); + const __m256i v1 = _mm256_loadu_si256( + reinterpret_cast(src + 32)); + const __m256i v2 = _mm256_loadu_si256( + reinterpret_cast(src + 64)); + const __m256i v3 = _mm256_loadu_si256( + reinterpret_cast(src + 96)); + __m256i * output = reinterpret_cast<__m256i *>(dst); + _mm256_stream_si256(output + 0, v0); + _mm256_stream_si256(output + 1, v1); + _mm256_stream_si256(output + 2, v2); + _mm256_stream_si256(output + 3, v3); + dst += 128U; + src += 128U; + size -= 128U; + } + + while (size >= 32U) + { + const __m256i value = _mm256_loadu_si256( + reinterpret_cast(src)); + _mm256_stream_si256(reinterpret_cast<__m256i *>(dst), value); + dst += 32U; + src += 32U; + size -= 32U; + } + + if (size) + memcpy(dst, src, size); + + _mm256_zeroupper(); + } + } +} diff --git a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp index 1f46a77d..93bdfb75 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPFrameTransport.cpp @@ -25,6 +25,7 @@ #include "transport/lgmp/CLGMPHost.h" #include "Atomic.h" #include "CDebug.h" +#include "WCWrite.h" #include #include @@ -1335,15 +1336,14 @@ void CLGMPFrameTransport::WriteFrameBuffer(unsigned frameIndex, void * src, { LGMPBuffer * fb = m_frameBuffer[frameIndex]; - memcpy( - reinterpret_cast( - reinterpret_cast(fb->data) + offset), - reinterpret_cast( - reinterpret_cast(src) + offset), - len); + WCWrite::Copy(fb->data + offset, + static_cast(src) + offset, len); if (setWritePos) + { + WCWrite::Flush(); fb->wp = (uint32_t)(offset + len); + } } void CLGMPFrameTransport::WriteFrameBufferRows(unsigned frameIndex, @@ -1355,7 +1355,7 @@ void CLGMPFrameTransport::WriteFrameBufferRows(unsigned frameIndex, uint8_t * source = static_cast(src) + offset; for (unsigned row = 0; row < rows; ++row) { - memcpy(dst, source, rowBytes); + WCWrite::Copy(dst, source, rowBytes); dst += pitch; source += pitch; } @@ -1365,5 +1365,6 @@ void CLGMPFrameTransport::FinalizeFrameBuffer(unsigned frameIndex) const { const KVMFRFrame * frame = m_frame[frameIndex]; LGMPBuffer * fb = m_frameBuffer[frameIndex]; + WCWrite::Flush(); fb->wp = frame->dataHeight * frame->pitch; }