Skip to content

Use scalar emulation of gather instruction for arg methods #65

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 20, 2023
Merged

Use scalar emulation of gather instruction for arg methods #65

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f2ca68b
Use scalar emulation of gather instruction for arg methods
Aug 21, 2023
4767a4c
Replace zmm_t with reg_t
Sep 7, 2023
22c2f02
Replace zmmi_t with regi_t
Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 18 additions & 21 deletions src/avx512-64bit-argsort.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ X86_SIMD_SORT_INLINE void argsort_8_64bit(type_t *arr, int64_t *arg, int32_t N)
{
using reg_t = typename vtype::reg_t;
typename vtype::opmask_t load_mask = (0x01 << N) - 0x01;
argzmm_t argzmm = argtype::maskz_loadu(load_mask, arg);
argreg_t argzmm = argtype::maskz_loadu(load_mask, arg);
reg_t arrzmm = vtype::template mask_i64gather<sizeof(type_t)>(
vtype::zmm_max(), load_mask, argzmm, arr);
arrzmm = sort_zmm_64bit<vtype, argtype>(arrzmm, argzmm);
Expand All @@ -83,9 +83,9 @@ X86_SIMD_SORT_INLINE void argsort_16_64bit(type_t *arr, int64_t *arg, int32_t N)
}
using reg_t = typename vtype::reg_t;
typename vtype::opmask_t load_mask = (0x01 << (N - 8)) - 0x01;
argzmm_t argzmm1 = argtype::loadu(arg);
argzmm_t argzmm2 = argtype::maskz_loadu(load_mask, arg + 8);
reg_t arrzmm1 = vtype::template i64gather<sizeof(type_t)>(argzmm1, arr);
argreg_t argzmm1 = argtype::loadu(arg);
argreg_t argzmm2 = argtype::maskz_loadu(load_mask, arg + 8);
reg_t arrzmm1 = vtype::i64gather(arr, arg);
reg_t arrzmm2 = vtype::template mask_i64gather<sizeof(type_t)>(
vtype::zmm_max(), load_mask, argzmm2, arr);
arrzmm1 = sort_zmm_64bit<vtype, argtype>(arrzmm1, argzmm1);
Expand All @@ -106,12 +106,12 @@ X86_SIMD_SORT_INLINE void argsort_32_64bit(type_t *arr, int64_t *arg, int32_t N)
using reg_t = typename vtype::reg_t;
using opmask_t = typename vtype::opmask_t;
reg_t arrzmm[4];
argzmm_t argzmm[4];
argreg_t argzmm[4];

X86_SIMD_SORT_UNROLL_LOOP(2)
for (int ii = 0; ii < 2; ++ii) {
argzmm[ii] = argtype::loadu(arg + 8 * ii);
arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
arrzmm[ii] = vtype::i64gather(arr, arg + 8 * ii);
arrzmm[ii] = sort_zmm_64bit<vtype, argtype>(arrzmm[ii], argzmm[ii]);
}

Expand Down Expand Up @@ -149,12 +149,12 @@ X86_SIMD_SORT_INLINE void argsort_64_64bit(type_t *arr, int64_t *arg, int32_t N)
using reg_t = typename vtype::reg_t;
using opmask_t = typename vtype::opmask_t;
reg_t arrzmm[8];
argzmm_t argzmm[8];
argreg_t argzmm[8];

X86_SIMD_SORT_UNROLL_LOOP(4)
for (int ii = 0; ii < 4; ++ii) {
argzmm[ii] = argtype::loadu(arg + 8 * ii);
arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
arrzmm[ii] = vtype::i64gather(arr, arg + 8 * ii);
arrzmm[ii] = sort_zmm_64bit<vtype, argtype>(arrzmm[ii], argzmm[ii]);
}

Expand Down Expand Up @@ -201,12 +201,12 @@ X86_SIMD_SORT_UNROLL_LOOP(4)
// using reg_t = typename vtype::reg_t;
// using opmask_t = typename vtype::opmask_t;
// reg_t arrzmm[16];
// argzmm_t argzmm[16];
// argreg_t argzmm[16];
//
//X86_SIMD_SORT_UNROLL_LOOP(8)
// for (int ii = 0; ii < 8; ++ii) {
// argzmm[ii] = argtype::loadu(arg + 8*ii);
// arrzmm[ii] = vtype::template i64gather<sizeof(type_t)>(argzmm[ii], arr);
// arrzmm[ii] = vtype::i64gather(argzmm[ii], arr);
// arrzmm[ii] = sort_zmm_64bit<vtype, argtype>(arrzmm[ii], argzmm[ii]);
// }
//
Expand Down Expand Up @@ -257,17 +257,14 @@ type_t get_pivot_64bit(type_t *arr,
// median of 8
int64_t size = (right - left) / 8;
using reg_t = typename vtype::reg_t;
// TODO: Use gather here too:
__m512i rand_index = _mm512_set_epi64(arg[left + size],
arg[left + 2 * size],
arg[left + 3 * size],
arg[left + 4 * size],
arg[left + 5 * size],
arg[left + 6 * size],
arg[left + 7 * size],
arg[left + 8 * size]);
reg_t rand_vec
= vtype::template i64gather<sizeof(type_t)>(rand_index, arr);
reg_t rand_vec = vtype::set(arr[arg[left + size]],
arr[arg[left + 2 * size]],
arr[arg[left + 3 * size]],
arr[arg[left + 4 * size]],
arr[arg[left + 5 * size]],
arr[arg[left + 6 * size]],
arr[arg[left + 7 * size]],
arr[arg[left + 8 * size]]);
// pivot will never be a nan, since there are no nan's!
reg_t sort = sort_zmm_64bit<vtype>(rand_vec);
return ((type_t *)&sort)[4];
Expand Down
Loading