mirror of
https://github.com/gnif/LookingGlass.git
synced 2026-08-22 15:11:31 +00:00
4477ac0de1
Add a reusable copy path for moving data from write-combined mappings into normal cacheable memory. Dispatch to MOVNTDQA-based SSE4.1 or AVX2 loads when the processor and OS support them, with a fenced fallback. Use the optimized path when staging inbound clipboard stream records from the IVSHMEM mapping.
142 lines
3.9 KiB
C++
142 lines
3.9 KiB
C++
/**
|
|
* Looking Glass
|
|
* Copyright © 2017-2026 The Looking Glass Authors
|
|
* https://looking-glass.io
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify it
|
|
* under the terms of the GNU General Public License as published by the Free
|
|
* Software Foundation; either version 2 of the License, or (at your option)
|
|
* any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
* more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License along
|
|
* with this program; if not, write to the Free Software Foundation, Inc., 59
|
|
* Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
*/
|
|
|
|
#include "WCCopy.h"
|
|
|
|
#include <immintrin.h>
|
|
#include <intrin.h>
|
|
#include <stdint.h>
|
|
#include <string.h>
|
|
|
|
namespace WCCopy
|
|
{
|
|
namespace Detail
|
|
{
|
|
void CopyAVX2(void * destination, const void * source, size_t size);
|
|
}
|
|
|
|
namespace
|
|
{
|
|
using CopyFn = void (*)(void *, const void *, size_t);
|
|
static constexpr unsigned __int64 AVX_XSTATE_MASK = 0x6U;
|
|
|
|
void CopyFallback(void * destination, const void * source, size_t size)
|
|
{
|
|
_mm_mfence();
|
|
memcpy(destination, source, size);
|
|
_mm_mfence();
|
|
}
|
|
|
|
#if defined(_M_IX86) || defined(_M_X64)
|
|
__declspec(noinline) void CopySSE41(
|
|
void * destination, const void * source, size_t size)
|
|
{
|
|
uint8_t * dst = static_cast<uint8_t *>(destination);
|
|
const uint8_t * src = static_cast<const uint8_t *>(source);
|
|
|
|
_mm_mfence();
|
|
|
|
const size_t prefix =
|
|
(16U - (reinterpret_cast<uintptr_t>(src) & 15U)) & 15U;
|
|
if (prefix)
|
|
{
|
|
const size_t copy = prefix < size ? prefix : size;
|
|
memcpy(dst, src, copy);
|
|
dst += copy;
|
|
src += copy;
|
|
size -= copy;
|
|
}
|
|
|
|
while (size >= 64U)
|
|
{
|
|
// MSVC's SSE4.1 intrinsic omits const from its input pointer.
|
|
__m128i * input = reinterpret_cast<__m128i *>(
|
|
const_cast<uint8_t *>(src));
|
|
__m128i * output = reinterpret_cast<__m128i *>(dst);
|
|
const __m128i v0 = _mm_stream_load_si128(input + 0);
|
|
const __m128i v1 = _mm_stream_load_si128(input + 1);
|
|
const __m128i v2 = _mm_stream_load_si128(input + 2);
|
|
const __m128i v3 = _mm_stream_load_si128(input + 3);
|
|
_mm_storeu_si128(output + 0, v0);
|
|
_mm_storeu_si128(output + 1, v1);
|
|
_mm_storeu_si128(output + 2, v2);
|
|
_mm_storeu_si128(output + 3, v3);
|
|
dst += 64U;
|
|
src += 64U;
|
|
size -= 64U;
|
|
}
|
|
|
|
while (size >= 16U)
|
|
{
|
|
const __m128i value = _mm_stream_load_si128(
|
|
reinterpret_cast<__m128i *>(const_cast<uint8_t *>(src)));
|
|
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst), value);
|
|
dst += 16U;
|
|
src += 16U;
|
|
size -= 16U;
|
|
}
|
|
|
|
if (size)
|
|
memcpy(dst, src, size);
|
|
|
|
_mm_mfence();
|
|
}
|
|
|
|
CopyFn SelectCopy()
|
|
{
|
|
int registers[4] = {};
|
|
__cpuid(registers, 0);
|
|
const int maximumLeaf = registers[0];
|
|
if (maximumLeaf < 1)
|
|
return &CopyFallback;
|
|
|
|
__cpuidex(registers, 1, 0);
|
|
const bool sse41 = (registers[2] & (1 << 19)) != 0;
|
|
const bool avx = (registers[2] & (1 << 28)) != 0;
|
|
const bool osxsave = (registers[2] & (1 << 27)) != 0;
|
|
|
|
if (maximumLeaf >= 7 && avx && osxsave &&
|
|
(_xgetbv(0) & AVX_XSTATE_MASK) == AVX_XSTATE_MASK)
|
|
{
|
|
__cpuidex(registers, 7, 0);
|
|
if ((registers[1] & (1 << 5)) != 0)
|
|
return &Detail::CopyAVX2;
|
|
}
|
|
|
|
return sse41 ? &CopySSE41 : &CopyFallback;
|
|
}
|
|
#else
|
|
CopyFn SelectCopy()
|
|
{
|
|
return &CopyFallback;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
void Copy(void * destination, const void * source, size_t size)
|
|
{
|
|
if (!size)
|
|
return;
|
|
|
|
static const CopyFn copy = SelectCopy();
|
|
copy(destination, source, size);
|
|
}
|
|
}
|