gwdev/Sync
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
master
Sync/kernel/memory/memcpy.c

2143 lines
70 KiB
C
Raw Permalink Normal View History

Add some public-domain memory manipulation files as a stand-in for my AVX functions. These are messy, but they let the kernel compile.
2019-07-24 19:24:38 +00:00
#include <kernel.h>
//
// This file provides a highly optimized version of memcpy.
// Overlapping memory regions are not supported by default: use memmove instead.
//
// NOTE: The discussion about microarchitecture in the memmove file applies to
// this memcpy, as well.
//
// ...If for some reason you absolutely, desperately need to use AVX_memcpy
// instead of AVX_memmove, and you need to use it on overlapping areas, enable
// the below definition. It will check for overlap and automatically redirect to
// AVX_memmove if overlap is found.
// #define OVERLAP_CHECK
//
#ifdef __clang__
#define __m128i_u __m128i
#define __m256i_u __m256i
#define __m512i_u __m512i
#endif
#ifdef __AVX512F__
#define BYTE_ALIGNMENT 0x3F // For 64-byte alignment
#elif __AVX__
#define BYTE_ALIGNMENT 0x1F // For 32-byte alignment
#else
#define BYTE_ALIGNMENT 0x0F // For 16-byte alignment
#endif
//
// USAGE INFORMATION:
//
// The "len" argument is "# of x bytes to copy," e.g. memcpy_512bit_u/a needs
// to know "how many multiples of 512 bit (64 bytes) to copy." The functions
// with byte sizes larger than their bit/8 sizes follow the same pattern:
// memcpy_512bit_512B_u/a needs to know how many multiples of 512 bytes to copy.
//
// The "numbytes" argument in AVX_memcpy and memcpy_large is just the total
// number of bytes to copy.
//
//-----------------------------------------------------------------------------
// Individual Functions:
//-----------------------------------------------------------------------------
// 16-bit (2 bytes at a time)
// Len is (# of total bytes/2), so it's "# of 16-bits"
void * memcpy_16bit(void *dest, const void *src, size_t len)
{
const uint16_t* s = (uint16_t*)src;
uint16_t* d = (uint16_t*)dest;
while (len--)
{
*d++ = *s++;
}
return dest;
}
// 32-bit (4 bytes at a time - 1 pixel in a 32-bit linear frame buffer)
// Len is (# of total bytes/4), so it's "# of 32-bits"
void * memcpy_32bit(void *dest, const void *src, size_t len)
{
const uint32_t* s = (uint32_t*)src;
uint32_t* d = (uint32_t*)dest;
while (len--)
{
*d++ = *s++;
}
return dest;
}
// 64-bit (8 bytes at a time - 2 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/8), so it's "# of 64-bits"
void * memcpy_64bit(void *dest, const void *src, size_t len)
{
const uint64_t* s = (uint64_t*)src;
uint64_t* d = (uint64_t*)dest;
while (len--)
{
*d++ = *s++;
}
return dest;
}
//-----------------------------------------------------------------------------
// SSE2 Unaligned:
//-----------------------------------------------------------------------------
// SSE2 (128-bit, 16 bytes at a time - 4 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/16), so it's "# of 128-bits"
void * memcpy_128bit_u(void *dest, const void *src, size_t len)
{
const __m128i_u* s = (__m128i_u*)src;
__m128i_u* d = (__m128i_u*)dest;
while (len--)
{
_mm_storeu_si128(d++, _mm_lddqu_si128(s++));
}
return dest;
}
// 32 bytes at a time
void * memcpy_128bit_32B_u(void *dest, const void *src, size_t len)
{
const __m128i_u* s = (__m128i_u*)src;
__m128i_u* d = (__m128i_u*)dest;
while (len--)
{
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 1
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 2
}
return dest;
}
// 64 bytes at a time
void * memcpy_128bit_64B_u(void *dest, const void *src, size_t len)
{
const __m128i_u* s = (__m128i_u*)src;
__m128i_u* d = (__m128i_u*)dest;
while (len--)
{
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 1
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 2
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 3
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 4
}
return dest;
}
// 128 bytes at a time
void * memcpy_128bit_128B_u(void *dest, const void *src, size_t len)
{
const __m128i_u* s = (__m128i_u*)src;
__m128i_u* d = (__m128i_u*)dest;
while (len--)
{
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 1
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 2
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 3
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 4
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 5
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 6
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 7
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 8
}
return dest;
}
// 256 bytes
void * memcpy_128bit_256B_u(void *dest, const void *src, size_t len)
{
const __m128i_u* s = (__m128i_u*)src;
__m128i_u* d = (__m128i_u*)dest;
while (len--)
{
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 1
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 2
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 3
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 4
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 5
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 6
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 7
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 8
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 9
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 10
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 11
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 12
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 13
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 14
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 15
_mm_storeu_si128(d++, _mm_lddqu_si128(s++)); // 16
}
return dest;
}
//-----------------------------------------------------------------------------
// AVX+ Unaligned:
//-----------------------------------------------------------------------------
// AVX (256-bit, 32 bytes at a time - 8 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/32), so it's "# of 256-bits"
// Sandybridge and Ryzen and up, Haswell and up for better performance
#ifdef __AVX__
void * memcpy_256bit_u(void *dest, const void *src, size_t len)
{
const __m256i_u* s = (__m256i_u*)src;
__m256i_u* d = (__m256i_u*)dest;
while (len--)
{
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++));
}
return dest;
}
// 64 bytes at a time
void * memcpy_256bit_64B_u(void *dest, const void *src, size_t len)
{
const __m256i_u* s = (__m256i_u*)src;
__m256i_u* d = (__m256i_u*)dest;
while (len--)
{
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 1
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 2
}
return dest;
}
// 128 bytes at a time
void * memcpy_256bit_128B_u(void *dest, const void *src, size_t len)
{
const __m256i_u* s = (__m256i_u*)src;
__m256i_u* d = (__m256i_u*)dest;
while (len--)
{
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 1
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 2
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 3
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 4
}
return dest;
}
// 256 bytes at a time
void * memcpy_256bit_256B_u(void *dest, const void *src, size_t len)
{
const __m256i_u* s = (__m256i_u*)src;
__m256i_u* d = (__m256i_u*)dest;
while (len--)
{
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 1
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 2
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 3
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 4
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 5
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 6
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 7
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 8
}
return dest;
}
// 512 bytes at a time, one load->store for every ymm register (there are 16)
void * memcpy_256bit_512B_u(void *dest, const void *src, size_t len)
{
const __m256i_u* s = (__m256i_u*)src;
__m256i_u* d = (__m256i_u*)dest;
while (len--)
{
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 1
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 2
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 3
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 4
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 5
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 6
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 7
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 8
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 9
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 10
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 11
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 12
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 13
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 14
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 15
_mm256_storeu_si256(d++, _mm256_lddqu_si256(s++)); // 16
}
return dest;
}
#endif
// AVX-512 (512-bit, 64 bytes at a time - 16 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/64), so it's "# of 512-bits"
// Requires AVX512F
#ifdef __AVX512F__
void * memcpy_512bit_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++));
}
return dest;
}
// 128 bytes at a time
void * memcpy_512bit_128B_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
}
return dest;
}
// 256 bytes at a time
void * memcpy_512bit_256B_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
}
return dest;
}
// 512 bytes (half a KB!!) at a time
void * memcpy_512bit_512B_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 5
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 6
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 7
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 8
}
return dest;
}
// 1024 bytes, or 1 kB
void * memcpy_512bit_1kB_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 5
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 6
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 7
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 8
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 9
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 10
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 11
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 12
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 13
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 14
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 15
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 16
}
return dest;
}
// 2048 bytes, or 2 kB
void * memcpy_512bit_2kB_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 5
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 6
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 7
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 8
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 9
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 10
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 11
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 12
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 13
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 14
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 15
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 16
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 17
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 18
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 19
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 20
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 21
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 22
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 23
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 24
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 25
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 26
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 27
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 28
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 29
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 30
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 31
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 32
}
return dest;
}
// 4096 bytes, or 4 kB
void * memcpy_512bit_4kB_u(void *dest, const void *src, size_t len)
{
const __m512i_u* s = (__m512i_u*)src;
__m512i_u* d = (__m512i_u*)dest;
while (len--)
{
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 5
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 6
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 7
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 8
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 9
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 10
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 11
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 12
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 13
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 14
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 15
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 16
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 17
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 18
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 19
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 20
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 21
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 22
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 23
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 24
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 25
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 26
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 27
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 28
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 29
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 30
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 31
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 32
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 1
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 2
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 3
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 4
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 5
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 6
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 7
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 8
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 9
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 10
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 11
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 12
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 13
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 14
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 15
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 16
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 17
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 18
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 19
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 20
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 21
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 22
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 23
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 24
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 25
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 26
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 27
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 28
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 29
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 30
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 31
_mm512_storeu_si512(d++, _mm512_loadu_si512(s++)); // 32
}
return dest;
}
#endif
// AVX-1024 support pending existence of the standard. It would be able to fit
// an entire 4 kB page in its registers at one time. Imagine that!
//-----------------------------------------------------------------------------
// SSE2 Aligned:
//-----------------------------------------------------------------------------
// SSE2 (128-bit, 16 bytes at a time - 4 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/16), so it's "# of 128-bits"
void * memcpy_128bit_a(void *dest, const void *src, size_t len)
{
const __m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_store_si128(d++, _mm_load_si128(s++));
}
return dest;
}
// 32 bytes at a time
void * memcpy_128bit_32B_a(void *dest, const void *src, size_t len)
{
const __m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_store_si128(d++, _mm_load_si128(s++)); // 1
_mm_store_si128(d++, _mm_load_si128(s++)); // 2
}
return dest;
}
// 64 bytes at a time
void * memcpy_128bit_64B_a(void *dest, const void *src, size_t len)
{
const __m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_store_si128(d++, _mm_load_si128(s++)); // 1
_mm_store_si128(d++, _mm_load_si128(s++)); // 2
_mm_store_si128(d++, _mm_load_si128(s++)); // 3
_mm_store_si128(d++, _mm_load_si128(s++)); // 4
}
return dest;
}
// 128 bytes at a time
void * memcpy_128bit_128B_a(void *dest, const void *src, size_t len)
{
const __m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_store_si128(d++, _mm_load_si128(s++)); // 1
_mm_store_si128(d++, _mm_load_si128(s++)); // 2
_mm_store_si128(d++, _mm_load_si128(s++)); // 3
_mm_store_si128(d++, _mm_load_si128(s++)); // 4
_mm_store_si128(d++, _mm_load_si128(s++)); // 5
_mm_store_si128(d++, _mm_load_si128(s++)); // 6
_mm_store_si128(d++, _mm_load_si128(s++)); // 7
_mm_store_si128(d++, _mm_load_si128(s++)); // 8
}
return dest;
}
// 256 bytes
void * memcpy_128bit_256B_a(void *dest, const void *src, size_t len)
{
const __m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_store_si128(d++, _mm_load_si128(s++)); // 1
_mm_store_si128(d++, _mm_load_si128(s++)); // 2
_mm_store_si128(d++, _mm_load_si128(s++)); // 3
_mm_store_si128(d++, _mm_load_si128(s++)); // 4
_mm_store_si128(d++, _mm_load_si128(s++)); // 5
_mm_store_si128(d++, _mm_load_si128(s++)); // 6
_mm_store_si128(d++, _mm_load_si128(s++)); // 7
_mm_store_si128(d++, _mm_load_si128(s++)); // 8
_mm_store_si128(d++, _mm_load_si128(s++)); // 9
_mm_store_si128(d++, _mm_load_si128(s++)); // 10
_mm_store_si128(d++, _mm_load_si128(s++)); // 11
_mm_store_si128(d++, _mm_load_si128(s++)); // 12
_mm_store_si128(d++, _mm_load_si128(s++)); // 13
_mm_store_si128(d++, _mm_load_si128(s++)); // 14
_mm_store_si128(d++, _mm_load_si128(s++)); // 15
_mm_store_si128(d++, _mm_load_si128(s++)); // 16
}
return dest;
}
//-----------------------------------------------------------------------------
// AVX+ Aligned:
//-----------------------------------------------------------------------------
// AVX (256-bit, 32 bytes at a time - 8 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/32), so it's "# of 256-bits"
// Sandybridge and Ryzen and up
#ifdef __AVX__
void * memcpy_256bit_a(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_store_si256(d++, _mm256_load_si256(s++));
}
return dest;
}
// 64 bytes at a time
void * memcpy_256bit_64B_a(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 1
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 2
}
return dest;
}
// 128 bytes at a time
void * memcpy_256bit_128B_a(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 1
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 2
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 3
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 4
}
return dest;
}
// 256 bytes at a time
void * memcpy_256bit_256B_a(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 1
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 2
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 3
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 4
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 5
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 6
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 7
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 8
}
return dest;
}
// 512 bytes
void * memcpy_256bit_512B_a(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 1
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 2
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 3
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 4
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 5
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 6
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 7
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 8
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 9
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 10
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 11
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 12
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 13
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 14
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 15
_mm256_store_si256(d++, _mm256_load_si256(s++)); // 16
}
return dest;
}
#endif
// AVX-512 (512-bit, 64 bytes at a time - 16 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/64), so it's "# of 512-bits"
// Requires AVX512F
#ifdef __AVX512F__
void * memcpy_512bit_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++));
}
return dest;
}
// 128 bytes at a time
void * memcpy_512bit_128B_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
}
return dest;
}
// 256 bytes at a time
void * memcpy_512bit_256B_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
}
return dest;
}
// 512 bytes (half a KB!!) at a time
void * memcpy_512bit_512B_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 5
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 6
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 7
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 8
}
return dest;
}
// 1024 bytes, or 1 kB
void * memcpy_512bit_1kB_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 5
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 6
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 7
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 8
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 9
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 10
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 11
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 12
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 13
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 14
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 15
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 16
}
return dest;
}
// 2048 bytes, or 2 kB
void * memcpy_512bit_2kB_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 5
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 6
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 7
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 8
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 9
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 10
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 11
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 12
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 13
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 14
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 15
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 16
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 17
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 18
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 19
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 20
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 21
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 22
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 23
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 24
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 25
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 26
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 27
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 28
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 29
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 30
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 31
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 32
}
return dest;
}
// 4096 bytes, or 4 kB
void * memcpy_512bit_4kB_a(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 5
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 6
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 7
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 8
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 9
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 10
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 11
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 12
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 13
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 14
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 15
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 16
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 17
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 18
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 19
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 20
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 21
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 22
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 23
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 24
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 25
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 26
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 27
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 28
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 29
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 30
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 31
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 32
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 1
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 2
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 3
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 4
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 5
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 6
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 7
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 8
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 9
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 10
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 11
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 12
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 13
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 14
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 15
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 16
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 17
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 18
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 19
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 20
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 21
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 22
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 23
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 24
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 25
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 26
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 27
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 28
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 29
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 30
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 31
_mm512_store_si512(d++, _mm512_load_si512(s++)); // 32
}
return dest;
}
#endif
//-----------------------------------------------------------------------------
// SSE4.1 Streaming:
//-----------------------------------------------------------------------------
// SSE4.1 (128-bit, 16 bytes at a time - 4 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/16), so it's "# of 128-bits"
void * memcpy_128bit_as(void *dest, const void *src, size_t len)
{
__m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_stream_si128(d++, _mm_stream_load_si128(s++));
}
_mm_sfence();
return dest;
}
// 32 bytes at a time
void * memcpy_128bit_32B_as(void *dest, const void *src, size_t len)
{
__m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 1
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 2
}
_mm_sfence();
return dest;
}
// 64 bytes at a time
void * memcpy_128bit_64B_as(void *dest, const void *src, size_t len)
{
__m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 1
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 2
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 3
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 4
}
_mm_sfence();
return dest;
}
// 128 bytes at a time
void * memcpy_128bit_128B_as(void *dest, const void *src, size_t len)
{
__m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 1
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 2
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 3
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 4
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 5
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 6
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 7
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 8
}
_mm_sfence();
return dest;
}
// 256 bytes
void * memcpy_128bit_256B_as(void *dest, const void *src, size_t len)
{
__m128i* s = (__m128i*)src;
__m128i* d = (__m128i*)dest;
while (len--)
{
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 1
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 2
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 3
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 4
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 5
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 6
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 7
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 8
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 9
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 10
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 11
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 12
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 13
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 14
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 15
_mm_stream_si128(d++, _mm_stream_load_si128(s++)); // 16
}
_mm_sfence();
return dest;
}
//-----------------------------------------------------------------------------
// AVX2+ Streaming:
//-----------------------------------------------------------------------------
// AVX2 (256-bit, 32 bytes at a time - 8 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/32), so it's "# of 256-bits"
// Haswell and Ryzen and up
#ifdef __AVX2__
void * memcpy_256bit_as(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++));
}
_mm_sfence();
return dest;
}
// 64 bytes at a time
void * memcpy_256bit_64B_as(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 1
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 2
}
_mm_sfence();
return dest;
}
// 128 bytes at a time
void * memcpy_256bit_128B_as(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 1
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 2
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 3
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 4
}
_mm_sfence();
return dest;
}
// 256 bytes at a time
void * memcpy_256bit_256B_as(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 1
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 2
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 3
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 4
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 5
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 6
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 7
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 8
}
_mm_sfence();
return dest;
}
// 512 bytes
void * memcpy_256bit_512B_as(void *dest, const void *src, size_t len)
{
const __m256i* s = (__m256i*)src;
__m256i* d = (__m256i*)dest;
while (len--)
{
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 1
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 2
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 3
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 4
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 5
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 6
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 7
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 8
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 9
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 10
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 11
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 12
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 13
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 14
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 15
_mm256_stream_si256(d++, _mm256_stream_load_si256(s++)); // 16
}
_mm_sfence();
return dest;
}
#endif
// AVX-512 (512-bit, 64 bytes at a time - 16 pixels in a 32-bit linear frame buffer)
// Len is (# of total bytes/64), so it's "# of 512-bits"
// Requires AVX512F
#ifdef __AVX512F__
void * memcpy_512bit_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++));
}
_mm_sfence();
return dest;
}
// 128 bytes at a time
void * memcpy_512bit_128B_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
}
_mm_sfence();
return dest;
}
// 256 bytes at a time
void * memcpy_512bit_256B_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
}
_mm_sfence();
return dest;
}
// 512 bytes (half a KB!!) at a time
void * memcpy_512bit_512B_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 5
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 6
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 7
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 8
}
_mm_sfence();
return dest;
}
// 1024 bytes, or 1 kB
void * memcpy_512bit_1kB_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 5
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 6
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 7
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 8
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 9
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 10
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 11
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 12
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 13
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 14
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 15
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 16
}
_mm_sfence();
return dest;
}
// 2048 bytes, or 2 kB
void * memcpy_512bit_2kB_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 5
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 6
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 7
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 8
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 9
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 10
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 11
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 12
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 13
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 14
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 15
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 16
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 17
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 18
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 19
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 20
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 21
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 22
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 23
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 24
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 25
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 26
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 27
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 28
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 29
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 30
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 31
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 32
}
_mm_sfence();
return dest;
}
// 4096 bytes, or 4 kB
void * memcpy_512bit_4kB_as(void *dest, const void *src, size_t len)
{
const __m512i* s = (__m512i*)src;
__m512i* d = (__m512i*)dest;
while (len--)
{
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 5
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 6
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 7
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 8
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 9
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 10
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 11
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 12
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 13
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 14
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 15
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 16
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 17
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 18
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 19
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 20
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 21
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 22
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 23
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 24
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 25
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 26
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 27
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 28
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 29
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 30
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 31
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 32
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 1
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 2
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 3
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 4
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 5
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 6
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 7
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 8
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 9
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 10
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 11
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 12
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 13
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 14
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 15
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 16
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 17
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 18
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 19
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 20
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 21
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 22
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 23
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 24
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 25
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 26
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 27
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 28
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 29
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 30
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 31
_mm512_stream_si512(d++, _mm512_stream_load_si512(s++)); // 32
}
_mm_sfence();
return dest;
}
#endif
//-----------------------------------------------------------------------------
// Dispatch Functions:
//-----------------------------------------------------------------------------
// Copy arbitrarily large amounts of data between 2 non-overlapping regions
void * memcpy_large(void *dest, void *src, size_t numbytes)
{
void * returnval = dest; // memcpy is supposed to return the destination
size_t offset = 0; // Offset size needs to match the size of a pointer
while(numbytes)
// The biggest sizes will go first for alignment. There's no benefit to using
// aligned loads over unaligned loads here, so all are unaligned.
// NOTE: Each memcpy has its own loop so that any one can be used individually.
{
if(numbytes < 2) // 1 byte
{
memcpy(dest, src, numbytes);
offset = numbytes & -1;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes = 0;
}
else if(numbytes < 4) // 2 bytes
{
memcpy_16bit(dest, src, numbytes >> 1);
offset = numbytes & -2;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1;
}
else if(numbytes < 8) // 4 bytes
{
memcpy_32bit(dest, src, numbytes >> 2);
offset = numbytes & -4;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 3;
}
else if(numbytes < 16) // 8 bytes
{
memcpy_64bit(dest, src, numbytes >> 3);
offset = numbytes & -8;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 7;
}
#ifdef __AVX512F__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_u(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_u(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_512bit_u(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_512bit_128B_u(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_512bit_256B_u(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else if(numbytes < 1024) // 512 bytes
{
memcpy_512bit_512B_u(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
else if(numbytes < 2048) // 1024 bytes (1 kB)
{
memcpy_512bit_1kB_u(dest, src, numbytes >> 10);
offset = numbytes & -1024;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1023;
}
else if(numbytes < 4096) // 2048 bytes (2 kB)
{
memcpy_512bit_2kB_u(dest, src, numbytes >> 11);
offset = numbytes & -2048;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 2047;
}
else // 4096 bytes (4 kB)
{
memcpy_512bit_4kB_u(dest, src, numbytes >> 12);
offset = numbytes & -4096;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 4095;
}
#elif __AVX__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_u(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_u(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_256bit_64B_u(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_256bit_128B_u(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_256bit_256B_u(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else // 512 bytes
{
memcpy_256bit_512B_u(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
#else // SSE2 only
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_u(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_128bit_32B_u(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_128bit_64B_u(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_128bit_128B_u(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else // 256 bytes
{
memcpy_128bit_256B_u(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
#endif
}
return returnval;
} // END MEMCPY LARGE, UNALIGNED
// Copy arbitrarily large amounts of data between 2 non-overlapping regions
// Aligned version
void * memcpy_large_a(void *dest, void *src, size_t numbytes)
{
void * returnval = dest; // memcpy is supposed to return the destination
size_t offset = 0; // Offset size needs to match the size of a pointer
while(numbytes)
// The biggest sizes will go first for alignment. There's no benefit to using
// aligned loads over unaligned loads here, so all are unaligned.
// NOTE: Each memcpy has its own loop so that any one can be used individually.
{
if(numbytes < 2) // 1 byte
{
memcpy(dest, src, numbytes);
offset = numbytes & -1;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes = 0;
}
else if(numbytes < 4) // 2 bytes
{
memcpy_16bit(dest, src, numbytes >> 1);
offset = numbytes & -2;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1;
}
else if(numbytes < 8) // 4 bytes
{
memcpy_32bit(dest, src, numbytes >> 2);
offset = numbytes & -4;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 3;
}
else if(numbytes < 16) // 8 bytes
{
memcpy_64bit(dest, src, numbytes >> 3);
offset = numbytes & -8;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 7;
}
#ifdef __AVX512F__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_a(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_a(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_512bit_a(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_512bit_128B_a(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_512bit_256B_a(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else if(numbytes < 1024) // 512 bytes
{
memcpy_512bit_512B_a(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
else if(numbytes < 2048) // 1024 bytes (1 kB)
{
memcpy_512bit_1kB_a(dest, src, numbytes >> 10);
offset = numbytes & -1024;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1023;
}
else if(numbytes < 4096) // 2048 bytes (2 kB)
{
memcpy_512bit_2kB_a(dest, src, numbytes >> 11);
offset = numbytes & -2048;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 2047;
}
else // 4096 bytes (4 kB)
{
memcpy_512bit_4kB_a(dest, src, numbytes >> 12);
offset = numbytes & -4096;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 4095;
}
#elif __AVX__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_a(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_a(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_256bit_64B_a(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_256bit_128B_a(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_256bit_256B_a(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else // 512 bytes
{
memcpy_256bit_512B_a(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
#else // SSE2 only
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_a(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_128bit_32B_a(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_128bit_64B_a(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_128bit_128B_a(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else // 256 bytes
{
memcpy_128bit_256B_a(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
#endif
}
return returnval;
} // END MEMCPY LARGE, ALIGNED
// Copy arbitrarily large amounts of data between 2 non-overlapping regions
// Aligned, streaming version
void * memcpy_large_as(void *dest, void *src, size_t numbytes)
{
void * returnval = dest; // memcpy is supposed to return the destination
size_t offset = 0; // Offset size needs to match the size of a pointer
while(numbytes)
// The biggest sizes will go first for alignment. There's no benefit to using
// aligned loads over unaligned loads here, so all are unaligned.
// NOTE: Each memcpy has its own loop so that any one can be used individually.
{
if(numbytes < 2) // 1 byte
{
memcpy(dest, src, numbytes);
offset = numbytes & -1;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes = 0;
}
else if(numbytes < 4) // 2 bytes
{
memcpy_16bit(dest, src, numbytes >> 1);
offset = numbytes & -2;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1;
}
else if(numbytes < 8) // 4 bytes
{
memcpy_32bit(dest, src, numbytes >> 2);
offset = numbytes & -4;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 3;
}
else if(numbytes < 16) // 8 bytes
{
memcpy_64bit(dest, src, numbytes >> 3);
offset = numbytes & -8;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 7;
}
#ifdef __AVX512F__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_as(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_as(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_512bit_as(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_512bit_128B_as(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_512bit_256B_as(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else if(numbytes < 1024) // 512 bytes
{
memcpy_512bit_512B_as(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
else if(numbytes < 2048) // 1024 bytes (1 kB)
{
memcpy_512bit_1kB_as(dest, src, numbytes >> 10);
offset = numbytes & -1024;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 1023;
}
else if(numbytes < 4096) // 2048 bytes (2 kB)
{
memcpy_512bit_2kB_as(dest, src, numbytes >> 11);
offset = numbytes & -2048;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 2047;
}
else // 4096 bytes (4 kB)
{
memcpy_512bit_4kB_as(dest, src, numbytes >> 12);
offset = numbytes & -4096;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 4095;
}
#elif __AVX2__
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_as(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_256bit_as(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_256bit_64B_as(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_256bit_128B_as(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else if(numbytes < 512) // 256 bytes
{
memcpy_256bit_256B_as(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
else // 512 bytes
{
memcpy_256bit_512B_as(dest, src, numbytes >> 9);
offset = numbytes & -512;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 511;
}
#else // SSE4.1 only
else if(numbytes < 32) // 16 bytes
{
memcpy_128bit_as(dest, src, numbytes >> 4);
offset = numbytes & -16;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 15;
}
else if(numbytes < 64) // 32 bytes
{
memcpy_128bit_32B_as(dest, src, numbytes >> 5);
offset = numbytes & -32;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 31;
}
else if(numbytes < 128) // 64 bytes
{
memcpy_128bit_64B_as(dest, src, numbytes >> 6);
offset = numbytes & -64;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 63;
}
else if(numbytes < 256) // 128 bytes
{
memcpy_128bit_128B_as(dest, src, numbytes >> 7);
offset = numbytes & -128;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 127;
}
else // 256 bytes
{
memcpy_128bit_256B_as(dest, src, numbytes >> 8);
offset = numbytes & -256;
dest = (char *)dest + offset;
src = (char *)src + offset;
numbytes &= 255;
}
#endif
}
return returnval;
} // END MEMCPY LARGE, ALIGNED, STREAMING
//-----------------------------------------------------------------------------
// Main Function:
//-----------------------------------------------------------------------------
// General-purpose function to call
void* memcpyAVX(void *dest, void *src, size_t numbytes)
{
void * returnval = dest;
if((char*)src == (char*)dest)
{
// Lol.
return returnval;
}
#ifdef OVERLAP_CHECK
// Overlap check
if(
(
(
(char*)dest > (char*)src
)
&&
(
(char*)dest < ((char*)src + numbytes)
)
)
||
(
(
(char*)src > (char*)dest
)
&&
(
(char*)src < ((char*)dest + numbytes)
)
)
) // Why didn't you just use memmove directly???
{
returnval = AVX_memmove(dest, src, numbytes);
return returnval;
}
#endif
if(
( ((uintptr_t)src & BYTE_ALIGNMENT) == 0 )
&&
( ((uintptr_t)dest & BYTE_ALIGNMENT) == 0 )
) // Check alignment
{
// This is the fastest case: src and dest are both cache line aligned.
if(numbytes > CACHESIZE)
{
memcpy_large_as(dest, src, numbytes);
}
else
{
memcpy_large_a(dest, src, numbytes); // Even if numbytes is small this'll work
}
}
else // Unaligned
{
size_t numbytes_to_align = (BYTE_ALIGNMENT + 1) - ((uintptr_t)dest & BYTE_ALIGNMENT);
if(numbytes > numbytes_to_align)
{
void * destoffset = (char*)dest + numbytes_to_align;
void * srcoffset = (char*)src + numbytes_to_align;
// Get to an aligned position.
// This may be a little slower, but since it'll be mostly scalar operations
// alignment doesn't matter. Worst case it uses two vector functions, and
// this process only needs to be done once per call if dest is unaligned.
memcpy_large(dest, src, numbytes_to_align);
// Now this should be faster since stores are aligned.
memcpy_large(destoffset, srcoffset, numbytes - numbytes_to_align); // Can't use streaming due to potential src misalignment
// On Haswell and up, cross cache line loads have a negligible penalty.
// Thus this will be slower on Sandy & Ivy Bridge, though Ivy Bridge will
// fare a little better (~2x, maybe?). Ryzen should generally fall somewhere
// inbetween Sandy Bridge and Haswell/Skylake on that front.
// NOTE: These are just rough theoretical estimates.
}
else // Small size
{
memcpy_large(dest, src, numbytes);
}
}
return returnval;
}
// AVX-1024+ support pending existence of the standard.
Reference in New Issue Copy Permalink