Optimise partial sort to use a single pass

mosullivan93 · mosullivan93 · commit 28abc6c1c9b9 · 2023-06-27T05:05:32.000+10:00
The original partial sort function was a combination of a QuickSelect
pass followed by a QuickSort pass. This can lead to unnecessary
comparisons and hence wasted CPU cycles. This update changes the
internal method used by both `avx512_qselect` and `avx512_partial_qsort`
so that they can both leverage the same logic. This is achieved by the
addition of a new parameter that indicates whether the function should
recurse into (and sort) the left partition (i.e. partially sort the
front of the array) or simply recurse into the partition containing the
desired element (i.e. select/position a single element).
diff --git a/src/avx512-16bit-common.h b/src/avx512-16bit-common.h
@@ -290,11 +290,12 @@ qsort_16bit_(type_t *arr, int64_t left, int64_t right, int64_t max_iters)
 }
 
 template <typename vtype, typename type_t>
-static void qselect_16bit_(type_t *arr,
-                           int64_t pos,
-                           int64_t left,
-                           int64_t right,
-                           int64_t max_iters)
+static void qselsort_16bit_(type_t *arr,
+                            int64_t pos,
+                            int64_t left,
+                            int64_t right,
+                            int64_t max_iters,
+                            bool sortleft)
 {
     /*
      * Resort to std::sort if quicksort isnt making any progress
@@ -316,10 +317,10 @@ static void qselect_16bit_(type_t *arr,
     type_t biggest = vtype::type_min();
     int64_t pivot_index = partition_avx512<vtype>(
             arr, left, right + 1, pivot, &smallest, &biggest);
-    if ((pivot != smallest) && (pos < pivot_index))
-        qselect_16bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1);
+    if ((pivot != smallest) && ((pos < pivot_index) || sortleft))
+        qselsort_16bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1, sortleft);
     else if ((pivot != biggest) && (pos >= pivot_index))
-        qselect_16bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1);
+        qselsort_16bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1, sortleft);
 }
 
 #endif // AVX512_16BIT_COMMON
diff --git a/src/avx512-16bit-qsort.hpp b/src/avx512-16bit-qsort.hpp
@@ -412,32 +412,32 @@ bool is_a_nan<uint16_t>(uint16_t elem)
 }
 
 template <>
-void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort(int16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_16bit_<zmm_vector<int16_t>, int16_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_16bit_<zmm_vector<int16_t>, int16_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
 template <>
-void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort(uint16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_16bit_<zmm_vector<uint16_t>, uint16_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_16bit_<zmm_vector<uint16_t>, uint16_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
-void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     int64_t indx_last_elem = arrsize - 1;
     if (UNLIKELY(hasnan)) {
          indx_last_elem = move_nans_to_end_of_array(arr, arrsize);
     }
     if (indx_last_elem >= k) {
-        qselect_16bit_<zmm_vector<float16>, uint16_t>(
-            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));
+        qselsort_16bit_<zmm_vector<float16>, uint16_t>(
+            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);
     }
 }
 
diff --git a/src/avx512-32bit-qsort.hpp b/src/avx512-32bit-qsort.hpp
@@ -657,11 +657,12 @@ qsort_32bit_(type_t *arr, int64_t left, int64_t right, int64_t max_iters)
 }
 
 template <typename vtype, typename type_t>
-static void qselect_32bit_(type_t *arr,
-                           int64_t pos,
-                           int64_t left,
-                           int64_t right,
-                           int64_t max_iters)
+static void qselsort_32bit_(type_t *arr,
+                            int64_t pos,
+                            int64_t left,
+                            int64_t right,
+                            int64_t max_iters,
+                            bool sortleft)
 {
     /*
      * Resort to std::sort if quicksort isnt making any progress
@@ -683,10 +684,10 @@ static void qselect_32bit_(type_t *arr,
     type_t biggest = vtype::type_min();
     int64_t pivot_index = partition_avx512_unrolled<vtype, 2>(
             arr, left, right + 1, pivot, &smallest, &biggest);
-    if ((pivot != smallest) && (pos < pivot_index))
-        qselect_32bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1);
+    if ((pivot != smallest) && ((pos < pivot_index) || sortleft))
+        qselsort_32bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1, sortleft);
     else if ((pivot != biggest) && (pos >= pivot_index))
-        qselect_32bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1);
+        qselsort_32bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1, sortleft);
 }
 
 X86_SIMD_SORT_INLINE int64_t replace_nan_with_inf(float *arr, int64_t arrsize)
@@ -715,33 +716,33 @@ replace_inf_with_nan(float *arr, int64_t arrsize, int64_t nan_count)
 }
 
 template <>
-void avx512_qselect<int32_t>(int32_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<int32_t>(int32_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_32bit_<zmm_vector<int32_t>, int32_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_32bit_<zmm_vector<int32_t>, int32_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
 template <>
-void avx512_qselect<uint32_t>(uint32_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<uint32_t>(uint32_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_32bit_<zmm_vector<uint32_t>, uint32_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_32bit_<zmm_vector<uint32_t>, uint32_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
 template <>
-void avx512_qselect<float>(float *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<float>(float *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     int64_t indx_last_elem = arrsize - 1;
     if (UNLIKELY(hasnan)) {
          indx_last_elem = move_nans_to_end_of_array(arr, arrsize);
     }
     if (indx_last_elem >= k) {
-        qselect_32bit_<zmm_vector<float>, float>(
-            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));
+        qselsort_32bit_<zmm_vector<float>, float>(
+            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);
     }
 }
 
diff --git a/src/avx512-64bit-qsort.hpp b/src/avx512-64bit-qsort.hpp
@@ -751,11 +751,12 @@ qsort_64bit_(type_t *arr, int64_t left, int64_t right, int64_t max_iters)
 }
 
 template <typename vtype, typename type_t>
-static void qselect_64bit_(type_t *arr,
-                           int64_t pos,
-                           int64_t left,
-                           int64_t right,
-                           int64_t max_iters)
+static void qselsort_64bit_(type_t *arr,
+                            int64_t pos,
+                            int64_t left,
+                            int64_t right,
+                            int64_t max_iters,
+                            bool sortleft)
 {
     /*
      * Resort to std::sort if quicksort isnt making any progress
@@ -777,40 +778,40 @@ static void qselect_64bit_(type_t *arr,
     type_t biggest = vtype::type_min();
     int64_t pivot_index = partition_avx512_unrolled<vtype, 8>(
             arr, left, right + 1, pivot, &smallest, &biggest);
-    if ((pivot != smallest) && (pos < pivot_index))
-        qselect_64bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1);
+    if ((pivot != smallest) && ((pos < pivot_index) || sortleft))
+        qselsort_64bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1, sortleft);
     else if ((pivot != biggest) && (pos >= pivot_index))
-        qselect_64bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1);
+        qselsort_64bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1, sortleft);
 }
 
 template <>
-void avx512_qselect<int64_t>(int64_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<int64_t>(int64_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_64bit_<zmm_vector<int64_t>, int64_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_64bit_<zmm_vector<int64_t>, int64_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
 template <>
-void avx512_qselect<uint64_t>(uint64_t *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<uint64_t>(uint64_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     if (arrsize > 1) {
-        qselect_64bit_<zmm_vector<uint64_t>, uint64_t>(
-                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));
+        qselsort_64bit_<zmm_vector<uint64_t>, uint64_t>(
+                arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);
     }
 }
 
 template <>
-void avx512_qselect<double>(double *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort<double>(double *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     int64_t indx_last_elem = arrsize - 1;
     if (UNLIKELY(hasnan)) {
          indx_last_elem = move_nans_to_end_of_array(arr, arrsize);
     }
     if (indx_last_elem >= k) {
-        qselect_64bit_<zmm_vector<double>, double>(
-            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));
+        qselsort_64bit_<zmm_vector<double>, double>(
+            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);
     }
 }
 
diff --git a/src/avx512-common-qsort.h b/src/avx512-common-qsort.h
@@ -100,19 +100,27 @@ void avx512_qsort(T *arr, int64_t arrsize);
 void avx512_qsort_fp16(uint16_t *arr, int64_t arrsize);
 
 template <typename T>
-void avx512_qselect(T *arr, int64_t k, int64_t arrsize, bool hasnan = false);
-void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan = false);
+void avx512_qselsort(T *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan);
+void avx512_qselsort_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan);
+
+template <typename T>
+inline void avx512_qselect(T *arr, int64_t k, int64_t arrsize, bool hasnan = false)
+{
+    avx512_qselsort<T>(arr, k, arrsize, false, hasnan);
+}
+inline void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan = false)
+{
+    avx512_qselsort_fp16(arr, k, arrsize, false, hasnan);
+}
 
 template <typename T>
 inline void avx512_partial_qsort(T *arr, int64_t k, int64_t arrsize, bool hasnan = false)
 {
-    avx512_qselect<T>(arr, k - 1, arrsize, hasnan);
-    avx512_qsort<T>(arr, k - 1);
+    avx512_qselsort<T>(arr, k - 1, arrsize, true, hasnan);
 }
 inline void avx512_partial_qsort_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan = false)
 {
-    avx512_qselect_fp16(arr, k - 1, arrsize, hasnan);
-    avx512_qsort_fp16(arr, k - 1);
+    avx512_qselsort_fp16(arr, k - 1, arrsize, true, hasnan);
 }
 
 // key-value sort routines
diff --git a/src/avx512fp16-16bit-qsort.hpp b/src/avx512fp16-16bit-qsort.hpp
@@ -153,15 +153,15 @@ bool is_a_nan<_Float16>(_Float16 elem)
 }
 
 template <>
-void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize, bool hasnan)
+void avx512_qselsort(_Float16 *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)
 {
     int64_t indx_last_elem = arrsize - 1;
     if (UNLIKELY(hasnan)) {
          indx_last_elem = move_nans_to_end_of_array(arr, arrsize);
     }
     if (indx_last_elem >= k) {
-        qselect_16bit_<zmm_vector<_Float16>, _Float16>(
-            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));
+        qselsort_16bit_<zmm_vector<_Float16>, _Float16>(
+            arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);
     }
 }
 

Original file line number	Diff line number	Diff line change
`@@ -412,32 +412,32 @@ bool is_a_nan<uint16_t>(uint16_t elem)`
`412`	`412`	`}`
`413`	`413`
`414`	`414`	`template <>`
`415`		`-void avx512_qselect(int16_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`415`	`+void avx512_qselsort(int16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`416`	`416`	`{`
`417`	`417`	`if (arrsize > 1) {`
`418`		`- qselect_16bit_<zmm_vector<int16_t>, int16_t>(`
`419`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`418`	`+ qselsort_16bit_<zmm_vector<int16_t>, int16_t>(`
	`419`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`420`	`420`	`}`
`421`	`421`	`}`
`422`	`422`
`423`	`423`	`template <>`
`424`		`-void avx512_qselect(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`424`	`+void avx512_qselsort(uint16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`425`	`425`	`{`
`426`	`426`	`if (arrsize > 1) {`
`427`		`- qselect_16bit_<zmm_vector<uint16_t>, uint16_t>(`
`428`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`427`	`+ qselsort_16bit_<zmm_vector<uint16_t>, uint16_t>(`
	`428`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`429`	`429`	`}`
`430`	`430`	`}`
`431`	`431`
`432`		`-void avx512_qselect_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`432`	`+void avx512_qselsort_fp16(uint16_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`433`	`433`	`{`
`434`	`434`	`int64_t indx_last_elem = arrsize - 1;`
`435`	`435`	`if (UNLIKELY(hasnan)) {`
`436`	`436`	`indx_last_elem = move_nans_to_end_of_array(arr, arrsize);`
`437`	`437`	`}`
`438`	`438`	`if (indx_last_elem >= k) {`
`439`		`- qselect_16bit_<zmm_vector<float16>, uint16_t>(`
`440`		`- arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));`
	`439`	`+ qselsort_16bit_<zmm_vector<float16>, uint16_t>(`
	`440`	`+ arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);`
`441`	`441`	`}`
`442`	`442`	`}`
`443`	`443`
Original file line number	Diff line number	Diff line change
`@@ -657,11 +657,12 @@ qsort_32bit_(type_t *arr, int64_t left, int64_t right, int64_t max_iters)`
`657`	`657`	`}`
`658`	`658`
`659`	`659`	`template <typename vtype, typename type_t>`
`660`		`-static void qselect_32bit_(type_t *arr,`
`661`		`- int64_t pos,`
`662`		`- int64_t left,`
`663`		`- int64_t right,`
`664`		`- int64_t max_iters)`
	`660`	`+static void qselsort_32bit_(type_t *arr,`
	`661`	`+ int64_t pos,`
	`662`	`+ int64_t left,`
	`663`	`+ int64_t right,`
	`664`	`+ int64_t max_iters,`
	`665`	`+ bool sortleft)`
`665`	`666`	`{`
`666`	`667`	`/*`
`667`	`668`	`* Resort to std::sort if quicksort isnt making any progress`
`@@ -683,10 +684,10 @@ static void qselect_32bit_(type_t *arr,`
`683`	`684`	`type_t biggest = vtype::type_min();`
`684`	`685`	`int64_t pivot_index = partition_avx512_unrolled<vtype, 2>(`
`685`	`686`	`arr, left, right + 1, pivot, &smallest, &biggest);`
`686`		`- if ((pivot != smallest) && (pos < pivot_index))`
`687`		`- qselect_32bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1);`
	`687`	`+ if ((pivot != smallest) && ((pos < pivot_index) \|\| sortleft))`
	`688`	`+ qselsort_32bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1, sortleft);`
`688`	`689`	`else if ((pivot != biggest) && (pos >= pivot_index))`
`689`		`- qselect_32bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1);`
	`690`	`+ qselsort_32bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1, sortleft);`
`690`	`691`	`}`
`691`	`692`
`692`	`693`	`X86_SIMD_SORT_INLINE int64_t replace_nan_with_inf(float *arr, int64_t arrsize)`
`@@ -715,33 +716,33 @@ replace_inf_with_nan(float *arr, int64_t arrsize, int64_t nan_count)`
`715`	`716`	`}`
`716`	`717`
`717`	`718`	`template <>`
`718`		`-void avx512_qselect<int32_t>(int32_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`719`	`+void avx512_qselsort<int32_t>(int32_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`719`	`720`	`{`
`720`	`721`	`if (arrsize > 1) {`
`721`		`- qselect_32bit_<zmm_vector<int32_t>, int32_t>(`
`722`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`722`	`+ qselsort_32bit_<zmm_vector<int32_t>, int32_t>(`
	`723`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`723`	`724`	`}`
`724`	`725`	`}`
`725`	`726`
`726`	`727`	`template <>`
`727`		`-void avx512_qselect<uint32_t>(uint32_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`728`	`+void avx512_qselsort<uint32_t>(uint32_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`728`	`729`	`{`
`729`	`730`	`if (arrsize > 1) {`
`730`		`- qselect_32bit_<zmm_vector<uint32_t>, uint32_t>(`
`731`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`731`	`+ qselsort_32bit_<zmm_vector<uint32_t>, uint32_t>(`
	`732`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`732`	`733`	`}`
`733`	`734`	`}`
`734`	`735`
`735`	`736`	`template <>`
`736`		`-void avx512_qselect<float>(float *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`737`	`+void avx512_qselsort<float>(float *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`737`	`738`	`{`
`738`	`739`	`int64_t indx_last_elem = arrsize - 1;`
`739`	`740`	`if (UNLIKELY(hasnan)) {`
`740`	`741`	`indx_last_elem = move_nans_to_end_of_array(arr, arrsize);`
`741`	`742`	`}`
`742`	`743`	`if (indx_last_elem >= k) {`
`743`		`- qselect_32bit_<zmm_vector<float>, float>(`
`744`		`- arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));`
	`744`	`+ qselsort_32bit_<zmm_vector<float>, float>(`
	`745`	`+ arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);`
`745`	`746`	`}`
`746`	`747`	`}`
`747`	`748`
Original file line number	Diff line number	Diff line change
`@@ -751,11 +751,12 @@ qsort_64bit_(type_t *arr, int64_t left, int64_t right, int64_t max_iters)`
`751`	`751`	`}`
`752`	`752`
`753`	`753`	`template <typename vtype, typename type_t>`
`754`		`-static void qselect_64bit_(type_t *arr,`
`755`		`- int64_t pos,`
`756`		`- int64_t left,`
`757`		`- int64_t right,`
`758`		`- int64_t max_iters)`
	`754`	`+static void qselsort_64bit_(type_t *arr,`
	`755`	`+ int64_t pos,`
	`756`	`+ int64_t left,`
	`757`	`+ int64_t right,`
	`758`	`+ int64_t max_iters,`
	`759`	`+ bool sortleft)`
`759`	`760`	`{`
`760`	`761`	`/*`
`761`	`762`	`* Resort to std::sort if quicksort isnt making any progress`
`@@ -777,40 +778,40 @@ static void qselect_64bit_(type_t *arr,`
`777`	`778`	`type_t biggest = vtype::type_min();`
`778`	`779`	`int64_t pivot_index = partition_avx512_unrolled<vtype, 8>(`
`779`	`780`	`arr, left, right + 1, pivot, &smallest, &biggest);`
`780`		`- if ((pivot != smallest) && (pos < pivot_index))`
`781`		`- qselect_64bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1);`
	`781`	`+ if ((pivot != smallest) && ((pos < pivot_index) \|\| sortleft))`
	`782`	`+ qselsort_64bit_<vtype>(arr, pos, left, pivot_index - 1, max_iters - 1, sortleft);`
`782`	`783`	`else if ((pivot != biggest) && (pos >= pivot_index))`
`783`		`- qselect_64bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1);`
	`784`	`+ qselsort_64bit_<vtype>(arr, pos, pivot_index, right, max_iters - 1, sortleft);`
`784`	`785`	`}`
`785`	`786`
`786`	`787`	`template <>`
`787`		`-void avx512_qselect<int64_t>(int64_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`788`	`+void avx512_qselsort<int64_t>(int64_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`788`	`789`	`{`
`789`	`790`	`if (arrsize > 1) {`
`790`		`- qselect_64bit_<zmm_vector<int64_t>, int64_t>(`
`791`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`791`	`+ qselsort_64bit_<zmm_vector<int64_t>, int64_t>(`
	`792`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`792`	`793`	`}`
`793`	`794`	`}`
`794`	`795`
`795`	`796`	`template <>`
`796`		`-void avx512_qselect<uint64_t>(uint64_t *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`797`	`+void avx512_qselsort<uint64_t>(uint64_t *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`797`	`798`	`{`
`798`	`799`	`if (arrsize > 1) {`
`799`		`- qselect_64bit_<zmm_vector<uint64_t>, uint64_t>(`
`800`		`- arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize));`
	`800`	`+ qselsort_64bit_<zmm_vector<uint64_t>, uint64_t>(`
	`801`	`+ arr, k, 0, arrsize - 1, 2 * (int64_t)log2(arrsize), sortleft);`
`801`	`802`	`}`
`802`	`803`	`}`
`803`	`804`
`804`	`805`	`template <>`
`805`		`-void avx512_qselect<double>(double *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`806`	`+void avx512_qselsort<double>(double *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`806`	`807`	`{`
`807`	`808`	`int64_t indx_last_elem = arrsize - 1;`
`808`	`809`	`if (UNLIKELY(hasnan)) {`
`809`	`810`	`indx_last_elem = move_nans_to_end_of_array(arr, arrsize);`
`810`	`811`	`}`
`811`	`812`	`if (indx_last_elem >= k) {`
`812`		`- qselect_64bit_<zmm_vector<double>, double>(`
`813`		`- arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));`
	`813`	`+ qselsort_64bit_<zmm_vector<double>, double>(`
	`814`	`+ arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);`
`814`	`815`	`}`
`815`	`816`	`}`
`816`	`817`
Original file line number	Diff line number	Diff line change
`@@ -153,15 +153,15 @@ bool is_a_nan<_Float16>(_Float16 elem)`
`153`	`153`	`}`
`154`	`154`
`155`	`155`	`template <>`
`156`		`-void avx512_qselect(_Float16 *arr, int64_t k, int64_t arrsize, bool hasnan)`
	`156`	`+void avx512_qselsort(_Float16 *arr, int64_t k, int64_t arrsize, bool sortleft, bool hasnan)`
`157`	`157`	`{`
`158`	`158`	`int64_t indx_last_elem = arrsize - 1;`
`159`	`159`	`if (UNLIKELY(hasnan)) {`
`160`	`160`	`indx_last_elem = move_nans_to_end_of_array(arr, arrsize);`
`161`	`161`	`}`
`162`	`162`	`if (indx_last_elem >= k) {`
`163`		`- qselect_16bit_<zmm_vector<_Float16>, _Float16>(`
`164`		`- arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem));`
	`163`	`+ qselsort_16bit_<zmm_vector<_Float16>, _Float16>(`
	`164`	`+ arr, k, 0, indx_last_elem, 2 * (int64_t)log2(indx_last_elem), sortleft);`
`165`	`165`	`}`
`166`	`166`	`}`
`167`	`167`