diff --git a/idd/LGCommon/LGCommon.vcxproj b/idd/LGCommon/LGCommon.vcxproj index df5265ca..408e8b97 100644 --- a/idd/LGCommon/LGCommon.vcxproj +++ b/idd/LGCommon/LGCommon.vcxproj @@ -70,6 +70,10 @@ + + + AdvancedVectorExtensions2 + @@ -85,6 +89,7 @@ + diff --git a/idd/LGCommon/LGCommon.vcxproj.filters b/idd/LGCommon/LGCommon.vcxproj.filters index ba0ef89e..8bc23dc1 100644 --- a/idd/LGCommon/LGCommon.vcxproj.filters +++ b/idd/LGCommon/LGCommon.vcxproj.filters @@ -27,6 +27,12 @@ Source Files + + Source Files + + + Source Files + @@ -68,5 +74,8 @@ Header Files + + Header Files + diff --git a/idd/LGCommon/WCCopy.cpp b/idd/LGCommon/WCCopy.cpp new file mode 100644 index 00000000..f89097b3 --- /dev/null +++ b/idd/LGCommon/WCCopy.cpp @@ -0,0 +1,141 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "WCCopy.h" + +#include +#include +#include +#include + +namespace WCCopy +{ + namespace Detail + { + void CopyAVX2(void * destination, const void * source, size_t size); + } + + namespace + { + using CopyFn = void (*)(void *, const void *, size_t); + static constexpr unsigned __int64 AVX_XSTATE_MASK = 0x6U; + + void CopyFallback(void * destination, const void * source, size_t size) + { + _mm_mfence(); + memcpy(destination, source, size); + _mm_mfence(); + } + +#if defined(_M_IX86) || defined(_M_X64) + __declspec(noinline) void CopySSE41( + void * destination, const void * source, size_t size) + { + uint8_t * dst = static_cast(destination); + const uint8_t * src = static_cast(source); + + _mm_mfence(); + + const size_t prefix = + (16U - (reinterpret_cast(src) & 15U)) & 15U; + if (prefix) + { + const size_t copy = prefix < size ? prefix : size; + memcpy(dst, src, copy); + dst += copy; + src += copy; + size -= copy; + } + + while (size >= 64U) + { + // MSVC's SSE4.1 intrinsic omits const from its input pointer. + __m128i * input = reinterpret_cast<__m128i *>( + const_cast(src)); + __m128i * output = reinterpret_cast<__m128i *>(dst); + const __m128i v0 = _mm_stream_load_si128(input + 0); + const __m128i v1 = _mm_stream_load_si128(input + 1); + const __m128i v2 = _mm_stream_load_si128(input + 2); + const __m128i v3 = _mm_stream_load_si128(input + 3); + _mm_storeu_si128(output + 0, v0); + _mm_storeu_si128(output + 1, v1); + _mm_storeu_si128(output + 2, v2); + _mm_storeu_si128(output + 3, v3); + dst += 64U; + src += 64U; + size -= 64U; + } + + while (size >= 16U) + { + const __m128i value = _mm_stream_load_si128( + reinterpret_cast<__m128i *>(const_cast(src))); + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), value); + dst += 16U; + src += 16U; + size -= 16U; + } + + if (size) + memcpy(dst, src, size); + + _mm_mfence(); + } + + CopyFn SelectCopy() + { + int registers[4] = {}; + __cpuid(registers, 0); + const int maximumLeaf = registers[0]; + if (maximumLeaf < 1) + return &CopyFallback; + + __cpuidex(registers, 1, 0); + const bool sse41 = (registers[2] & (1 << 19)) != 0; + const bool avx = (registers[2] & (1 << 28)) != 0; + const bool osxsave = (registers[2] & (1 << 27)) != 0; + + if (maximumLeaf >= 7 && avx && osxsave && + (_xgetbv(0) & AVX_XSTATE_MASK) == AVX_XSTATE_MASK) + { + __cpuidex(registers, 7, 0); + if ((registers[1] & (1 << 5)) != 0) + return &Detail::CopyAVX2; + } + + return sse41 ? &CopySSE41 : &CopyFallback; + } +#else + CopyFn SelectCopy() + { + return &CopyFallback; + } +#endif + } + + void Copy(void * destination, const void * source, size_t size) + { + if (!size) + return; + + static const CopyFn copy = SelectCopy(); + copy(destination, source, size); + } +} diff --git a/idd/LGCommon/WCCopy.h b/idd/LGCommon/WCCopy.h new file mode 100644 index 00000000..e88dd70d --- /dev/null +++ b/idd/LGCommon/WCCopy.h @@ -0,0 +1,30 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#pragma once + +#include + +namespace WCCopy +{ + // Copy from write-combined memory into normal cacheable memory. The source + // and destination ranges must not overlap. + void Copy(void * destination, const void * source, size_t size); +} diff --git a/idd/LGCommon/WCCopyAVX2.cpp b/idd/LGCommon/WCCopyAVX2.cpp new file mode 100644 index 00000000..89e8baf1 --- /dev/null +++ b/idd/LGCommon/WCCopyAVX2.cpp @@ -0,0 +1,85 @@ +/** + * Looking Glass + * Copyright © 2017-2026 The Looking Glass Authors + * https://looking-glass.io + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#include "WCCopy.h" + +#include +#include +#include + +namespace WCCopy +{ + namespace Detail + { + __declspec(noinline) void CopyAVX2( + void * destination, const void * source, size_t size) + { + uint8_t * dst = static_cast(destination); + const uint8_t * src = static_cast(source); + + _mm_mfence(); + + const size_t prefix = + (32U - (reinterpret_cast(src) & 31U)) & 31U; + if (prefix) + { + const size_t copy = prefix < size ? prefix : size; + memcpy(dst, src, copy); + dst += copy; + src += copy; + size -= copy; + } + + while (size >= 128U) + { + const __m256i * input = + reinterpret_cast(src); + __m256i * output = reinterpret_cast<__m256i *>(dst); + const __m256i v0 = _mm256_stream_load_si256(input + 0); + const __m256i v1 = _mm256_stream_load_si256(input + 1); + const __m256i v2 = _mm256_stream_load_si256(input + 2); + const __m256i v3 = _mm256_stream_load_si256(input + 3); + _mm256_storeu_si256(output + 0, v0); + _mm256_storeu_si256(output + 1, v1); + _mm256_storeu_si256(output + 2, v2); + _mm256_storeu_si256(output + 3, v3); + dst += 128U; + src += 128U; + size -= 128U; + } + + while (size >= 32U) + { + const __m256i value = _mm256_stream_load_si256( + reinterpret_cast(src)); + _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), value); + dst += 32U; + src += 32U; + size -= 32U; + } + + if (size) + memcpy(dst, src, size); + + _mm_mfence(); + _mm256_zeroupper(); + } + } +} diff --git a/idd/LGIdd/transport/lgmp/CLGMPClipboardTransport.cpp b/idd/LGIdd/transport/lgmp/CLGMPClipboardTransport.cpp index 5dc09abb..a1d199e5 100644 --- a/idd/LGIdd/transport/lgmp/CLGMPClipboardTransport.cpp +++ b/idd/LGIdd/transport/lgmp/CLGMPClipboardTransport.cpp @@ -26,6 +26,7 @@ #include "CDebug.h" #include "Seq.h" +#include "WCCopy.h" #include "transport/lgmp/CLGMPHost.h" #include @@ -637,7 +638,7 @@ bool CLGMPClipboardTransport::QueueStreamTarget( (m_streamTargetHead + m_streamTargetCount) % STREAM_TARGET_COUNT]; target.record = record; if (record.length) - memcpy(target.data, data, record.length); + WCCopy::Copy(target.data, data, record.length); ++m_streamTargetCount; return true; }