| File: | root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu/arch/x86/repack-x86.cpp |
| Warning: | line 3566, column 5 Value stored to 'scalesmask1' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | #define GGML_COMMON_IMPL_CPP |
| 2 | #define GGML_COMMON_DECL_CPP |
| 3 | #include "ggml-common.h" |
| 4 | #include "ggml-backend-impl.h" |
| 5 | |
| 6 | #include "ggml-impl.h" |
| 7 | #include "ggml-cpu.h" |
| 8 | #include "ggml-cpu-impl.h" |
| 9 | #include "simd-mappings.h" |
| 10 | #include "traits.h" |
| 11 | |
| 12 | #include <cmath> |
| 13 | #include <cstring> |
| 14 | #include <cassert> |
| 15 | #include <cstdlib> // for qsort |
| 16 | #include <cstdio> // for GGML_ASSERT |
| 17 | |
| 18 | #define GGML_CPU_CLANG_WORKAROUND |
| 19 | #include "../../repack.h" |
| 20 | |
| 21 | #if defined(__GNUC__4) |
| 22 | #pragma GCC diagnostic ignored "-Woverlength-strings" |
| 23 | #endif |
| 24 | |
| 25 | #define UNUSEDGGML_UNUSED GGML_UNUSED |
| 26 | |
| 27 | #if defined(__AVX__1) |
| 28 | #if defined(__F16C__1) |
| 29 | #if defined(__AVX512F__) |
| 30 | #define GGML_F32Cx8x2_LOAD(x, y) _mm512_cvtph_ps(_mm256_set_m128i(_mm_loadu_si128((const __m128i *)(y)), _mm_loadu_si128((const __m128i *)(x)))) |
| 31 | #define GGML_F32Cx16_REPEAT_LOAD(x) _mm512_cvtph_ps(_mm256_set_m128i(x, x)) |
| 32 | #endif |
| 33 | // the _mm256_cvt intrinsics require F16C |
| 34 | #define GGML_F32Cx8_LOAD(x)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x))) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x))) |
| 35 | #define GGML_F32Cx8_REPEAT_LOAD(x, loadMask)_mm256_cvtph_ps(((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i )(_mm_maskload_epi32((int const*)(x), loadMask)), (int)(68))) ) _mm256_cvtph_ps(_mm_shuffle_epi32(_mm_maskload_epi32((int const*)(x), loadMask), 68)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(_mm_maskload_epi32 ((int const*)(x), loadMask)), (int)(68)))) |
| 36 | #define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask)_mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask)) _mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask)) |
| 37 | #else |
| 38 | #if defined(__AVX512F__) |
| 39 | static inline __m512 __avx512_f32cx8x2_load(ggml_fp16_t *x, ggml_fp16_t *y) { |
| 40 | float tmp[16]; |
| 41 | |
| 42 | for (int i = 0; i < 8; i++) { |
| 43 | tmp[i] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]); |
| 44 | } |
| 45 | |
| 46 | for (int i = 0; i < 8; i++) { |
| 47 | tmp[i + 8] = GGML_CPU_FP16_TO_FP32(y[i])ggml_lookup_fp16_to_fp32(y[i]); |
| 48 | } |
| 49 | |
| 50 | return _mm512_loadu_ps(tmp); |
| 51 | } |
| 52 | static inline __m512 __avx512_repeat_f32cx16_load(__m128i x) { |
| 53 | float tmp[16]; |
| 54 | uint16_t tmphalf[8]; |
| 55 | _mm_storeu_si128((__m128i*)tmphalf, x); |
| 56 | |
| 57 | for (int i = 0; i < 4; i++) { |
| 58 | tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i])ggml_lookup_fp16_to_fp32(tmphalf[i]); |
| 59 | tmp[i + 4] = GGML_CPU_FP16_TO_FP32(tmphalf[i])ggml_lookup_fp16_to_fp32(tmphalf[i]); |
| 60 | tmp[i + 8] = GGML_CPU_FP16_TO_FP32(tmphalf[i])ggml_lookup_fp16_to_fp32(tmphalf[i]); |
| 61 | tmp[i + 12] = GGML_CPU_FP16_TO_FP32(tmphalf[i])ggml_lookup_fp16_to_fp32(tmphalf[i]); |
| 62 | } |
| 63 | |
| 64 | return _mm512_loadu_ps(tmp); |
| 65 | } |
| 66 | #endif |
| 67 | static inline __m256 __avx_f32cx8_load(ggml_fp16_t *x) { |
| 68 | float tmp[8]; |
| 69 | |
| 70 | for (int i = 0; i < 8; i++) { |
| 71 | tmp[i] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]); |
| 72 | } |
| 73 | |
| 74 | return _mm256_loadu_ps(tmp); |
| 75 | } |
| 76 | static inline __m256 __avx_repeat_f32cx8_load(ggml_fp16_t *x) { |
| 77 | float tmp[8]; |
| 78 | |
| 79 | for (int i = 0; i < 4; i++) { |
| 80 | tmp[i] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]); |
| 81 | tmp[i + 4] = GGML_CPU_FP16_TO_FP32(x[i])ggml_lookup_fp16_to_fp32(x[i]); |
| 82 | } |
| 83 | |
| 84 | return _mm256_loadu_ps(tmp); |
| 85 | } |
| 86 | static inline __m256 __avx_rearranged_f32cx8_load(ggml_fp16_t *x, __m128i arrangeMask) { |
| 87 | uint16_t tmphalf[8]; |
| 88 | float tmp[8]; |
| 89 | |
| 90 | _mm_storeu_si128((__m128i*)tmphalf, _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask)); |
| 91 | for (int i = 0; i < 8; i++) { |
| 92 | tmp[i] = GGML_CPU_FP16_TO_FP32(tmphalf[i])ggml_lookup_fp16_to_fp32(tmphalf[i]); |
| 93 | } |
| 94 | |
| 95 | return _mm256_loadu_ps(tmp); |
| 96 | } |
| 97 | |
| 98 | #define GGML_F32Cx8_LOAD(x)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x))) __avx_f32cx8_load(x) |
| 99 | #define GGML_F32Cx8_REPEAT_LOAD(x, loadMask)_mm256_cvtph_ps(((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i )(_mm_maskload_epi32((int const*)(x), loadMask)), (int)(68))) ) __avx_repeat_f32cx8_load(x) |
| 100 | #define GGML_F32Cx8_REARRANGE_LOAD(x, arrangeMask)_mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) x), arrangeMask)) __avx_rearranged_f32cx8_load(x, arrangeMask) |
| 101 | #if defined(__AVX512F__) |
| 102 | #define GGML_F32Cx8x2_LOAD(x, y) __avx512_f32cx8x2_load(x, y) |
| 103 | #define GGML_F32Cx16_REPEAT_LOAD(x) __avx512_repeat_f32cx16_load(x) |
| 104 | #endif |
| 105 | #endif |
| 106 | #endif |
| 107 | |
| 108 | static inline int nearest_int(float fval) { |
| 109 | assert(fabsf(fval) <= 4194303.f)(static_cast <bool> (fabsf(fval) <= 4194303.f) ? void (0) : __assert_fail ("fabsf(fval) <= 4194303.f", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 110 | float val = fval + 12582912.f; |
| 111 | int i; memcpy(&i, &val, sizeof(int)); |
| 112 | return (i & 0x007fffff) - 0x00400000; |
| 113 | } |
| 114 | |
| 115 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 116 | #if defined(__AVX512F__) |
| 117 | // add int16_t pairwise and return as 512 bit int vector, then add the accumulator |
| 118 | static inline __m512i sum_i16_pairs_acc_int32x16(const __m512i acc, const __m512i x) { |
| 119 | const __m512i ones = _mm512_set1_epi16(1); |
| 120 | return _mm512_add_epi32(acc, _mm512_madd_epi16(ones, x)); |
| 121 | } |
| 122 | |
| 123 | static inline __m512i mul_sum_us8_pairs_acc_int32x16(const __m512i acc, const __m512i ax, const __m512i sy) { |
| 124 | #if defined(__AVX512VNNI__) |
| 125 | return _mm512_dpbusd_epi32(acc, ax, sy); |
| 126 | #else |
| 127 | // Perform multiplication and create 16-bit values |
| 128 | const __m512i dot = _mm512_maddubs_epi16(ax, sy); |
| 129 | return sum_i16_pairs_acc_int32x16(acc, dot); |
| 130 | #endif |
| 131 | } |
| 132 | |
| 133 | // multiply int8_t, add results pairwise twice and return as 512 bit int vector，then add the accumulator |
| 134 | static inline __m512i mul_sum_i8_pairs_acc_int32x16(const __m512i acc, const __m512i x, const __m512i y) { |
| 135 | const __m512i zero = _mm512_setzero_si512(); |
| 136 | // Get absolute values of x vectors |
| 137 | const __m512i ax = _mm512_abs_epi8(x); |
| 138 | // Sign the values of the y vectors |
| 139 | __mmask64 blt0 = _mm512_movepi8_mask(x); |
| 140 | const __m512i sy = _mm512_mask_sub_epi8(y, blt0, zero, y); |
| 141 | return mul_sum_us8_pairs_acc_int32x16(acc, ax, sy); |
| 142 | } |
| 143 | #endif |
| 144 | |
| 145 | // add int16_t pairwise and return as 256 bit int vector, then add the accumulator |
| 146 | static inline __m256i sum_i16_pairs_acc_int32x8(const __m256i acc, const __m256i x) { |
| 147 | const __m256i ones = _mm256_set1_epi16(1); |
| 148 | return _mm256_add_epi32(acc, _mm256_madd_epi16(ones, x)); |
| 149 | } |
| 150 | |
| 151 | static inline __m256i mul_sum_us8_pairs_acc_int32x8(const __m256i acc, const __m256i ax, const __m256i sy) { |
| 152 | #if defined(__AVX512VNNI__) && defined(__AVX512VL__) |
| 153 | return _mm256_dpbusd_epi32(acc, ax, sy)((__m256i)__builtin_ia32_vpdpbusd256((__v8si)(acc), (__v32qu) (ax), (__v32qi)(sy))); |
| 154 | #elif defined(__AVXVNNI__) |
| 155 | return _mm256_dpbusd_avx_epi32(acc, ax, sy); |
| 156 | #else |
| 157 | // Perform multiplication and create 16-bit values |
| 158 | const __m256i dot = _mm256_maddubs_epi16(ax, sy); |
| 159 | return sum_i16_pairs_acc_int32x8(acc, dot); |
| 160 | #endif |
| 161 | } |
| 162 | |
| 163 | // Integer variant of the function defined in ggml-quants.c |
| 164 | // multiply int8_t, add results pairwise twice and return as 256 bit int vector, then add the accumulator |
| 165 | static inline __m256i mul_sum_i8_pairs_acc_int32x8(const __m256i acc, const __m256i x, const __m256i y) { |
| 166 | #if defined(__AVXVNNIINT8__) |
| 167 | return _mm256_dpbssd_epi32(acc, x, y)((__m256i)__builtin_ia32_vpdpbssd256((__v8si)(acc), (__v32qi) (x), (__v32qi)(y))); |
| 168 | #else |
| 169 | // Get absolute values of x vectors |
| 170 | const __m256i ax = _mm256_sign_epi8(x, x); |
| 171 | // Sign the values of the y vectors |
| 172 | const __m256i sy = _mm256_sign_epi8(y, x); |
| 173 | return mul_sum_us8_pairs_acc_int32x8(acc, ax, sy); |
| 174 | #endif |
| 175 | } |
| 176 | #endif |
| 177 | |
| 178 | void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT__restrict__ x, void * GGML_RESTRICT__restrict__ vy, int64_t k) { |
| 179 | assert(QK8_0 == 32)(static_cast <bool> (32 == 32) ? void (0) : __assert_fail ("QK8_0 == 32", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 180 | assert(k % QK8_0 == 0)(static_cast <bool> (k % 32 == 0) ? void (0) : __assert_fail ("k % QK8_0 == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 181 | const int nb = k / QK8_032; |
| 182 | |
| 183 | block_q8_0x4 * GGML_RESTRICT__restrict__ y = (block_q8_0x4 *) vy; |
| 184 | |
| 185 | #if defined(__AVX2__1) || defined(__AVX__1) |
| 186 | float id[4]; |
| 187 | __m256 srcv[4][4]; |
| 188 | __m256 idvec[4]; |
| 189 | |
| 190 | for (int i = 0; i < nb; i++) { |
| 191 | for (int row_iter = 0; row_iter < 4; row_iter++) { |
| 192 | // Load elements into 4 AVX vectors |
| 193 | __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 32 ); |
| 194 | __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 8 ); |
| 195 | __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 16 ); |
| 196 | __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 32 + 24 ); |
| 197 | |
| 198 | // Compute max(abs(e)) for the block |
| 199 | const __m256 signBit = _mm256_set1_ps( -0.0f ); |
| 200 | __m256 maxAbs = _mm256_andnot_ps( signBit, v0 ); |
| 201 | maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v1 ) ); |
| 202 | maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v2 ) ); |
| 203 | maxAbs = _mm256_max_ps( maxAbs, _mm256_andnot_ps( signBit, v3 ) ); |
| 204 | |
| 205 | __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 )((__m128)__builtin_ia32_vextractf128_ps256((__v8sf)(__m256)(maxAbs ), (int)(1))), _mm256_castps256_ps128( maxAbs ) ); |
| 206 | max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); |
| 207 | max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); |
| 208 | const float maxScalar = _mm_cvtss_f32( max4 ); |
| 209 | |
| 210 | // Divided by 127.f to mirror results in quantize_row_q8_0 |
| 211 | const float d = maxScalar / 127.f; |
| 212 | id[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; //d ? 1.0f / d : 0.0f; |
| 213 | |
| 214 | // Store the scale for the individual block |
| 215 | y[i].d[row_iter] = GGML_CPU_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 216 | |
| 217 | // Store the values in blocks of eight values - Aim is to use these later for block interleaving |
| 218 | srcv[row_iter][0] = v0; |
| 219 | srcv[row_iter][1] = v1; |
| 220 | srcv[row_iter][2] = v2; |
| 221 | srcv[row_iter][3] = v3; |
| 222 | idvec[row_iter] = _mm256_set1_ps(id[row_iter]); |
| 223 | } |
| 224 | |
| 225 | // The loop iterates four times - The aim is to get 4 corresponding chunks of eight bytes from the original weight blocks that are interleaved |
| 226 | for (int j = 0; j < 4; j++) { |
| 227 | // Apply the multiplier |
| 228 | __m256 v0 = _mm256_mul_ps(srcv[0][j], idvec[0]); |
| 229 | __m256 v1 = _mm256_mul_ps(srcv[1][j], idvec[1]); |
| 230 | __m256 v2 = _mm256_mul_ps(srcv[2][j], idvec[2]); |
| 231 | __m256 v3 = _mm256_mul_ps(srcv[3][j], idvec[3]); |
| 232 | |
| 233 | // Round to nearest integer |
| 234 | v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v0), ((0x0000U )))); |
| 235 | v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v1), ((0x0000U )))); |
| 236 | v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v2), ((0x0000U )))); |
| 237 | v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v3), ((0x0000U )))); |
| 238 | |
| 239 | // Convert floats to integers |
| 240 | __m256i i0 = _mm256_cvtps_epi32( v0 ); |
| 241 | __m256i i1 = _mm256_cvtps_epi32( v1 ); |
| 242 | __m256i i2 = _mm256_cvtps_epi32( v2 ); |
| 243 | __m256i i3 = _mm256_cvtps_epi32( v3 ); |
| 244 | |
| 245 | #if defined(__AVX2__1) |
| 246 | // Convert int32 to int16 |
| 247 | i0 = _mm256_packs_epi32( i0, i1 ); |
| 248 | i2 = _mm256_packs_epi32( i2, i3 ); |
| 249 | // Convert int16 to int8 |
| 250 | i0 = _mm256_packs_epi16( i0, i2 ); |
| 251 | |
| 252 | // Permute and store the quantized weights in the required order after the pack instruction |
| 253 | const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); |
| 254 | i0 = _mm256_permutevar8x32_epi32( i0, perm ); |
| 255 | |
| 256 | _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0); |
| 257 | #else |
| 258 | // Since we don't have in AVX some necessary functions, |
| 259 | // we split the registers in half and call AVX2 analogs from SSE |
| 260 | __m128i ni0 = _mm256_castsi256_si128( i0 ); |
| 261 | __m128i ni1 = _mm256_extractf128_si256( i0, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (i0), (int)(1))); |
| 262 | __m128i ni2 = _mm256_castsi256_si128( i1 ); |
| 263 | __m128i ni3 = _mm256_extractf128_si256( i1, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (i1), (int)(1))); |
| 264 | __m128i ni4 = _mm256_castsi256_si128( i2 ); |
| 265 | __m128i ni5 = _mm256_extractf128_si256( i2, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (i2), (int)(1))); |
| 266 | __m128i ni6 = _mm256_castsi256_si128( i3 ); |
| 267 | __m128i ni7 = _mm256_extractf128_si256( i3, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (i3), (int)(1))); |
| 268 | |
| 269 | // Convert int32 to int16 |
| 270 | ni0 = _mm_packs_epi32( ni0, ni1 ); |
| 271 | ni2 = _mm_packs_epi32( ni2, ni3 ); |
| 272 | ni4 = _mm_packs_epi32( ni4, ni5 ); |
| 273 | ni6 = _mm_packs_epi32( ni6, ni7 ); |
| 274 | // Convert int16 to int8 |
| 275 | ni0 = _mm_packs_epi16( ni0, ni2 ); |
| 276 | ni4 = _mm_packs_epi16( ni4, ni6 ); |
| 277 | _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j), ni0); |
| 278 | _mm_storeu_si128((__m128i *)(y[i].qs + 32 * j + 16), ni4); |
| 279 | #endif |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | #else |
| 284 | UNUSED(nb)(void)(nb); |
| 285 | UNUSED(y)(void)(y); |
| 286 | ggml_quantize_mat_q8_0_4x8_generic(x, vy, k); |
| 287 | #endif |
| 288 | } |
| 289 | |
| 290 | void ggml_quantize_mat_q8_K_4x8(const float * GGML_RESTRICT__restrict__ x, void * GGML_RESTRICT__restrict__ vy, int64_t k) { |
| 291 | assert(QK_K == 256)(static_cast <bool> (256 == 256) ? void (0) : __assert_fail ("QK_K == 256", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 292 | assert(k % QK_K == 0)(static_cast <bool> (k % 256 == 0) ? void (0) : __assert_fail ("k % QK_K == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 293 | const int nb = k / QK_K256; |
| 294 | |
| 295 | block_q8_Kx4 * GGML_RESTRICT__restrict__ y = (block_q8_Kx4 *) vy; |
| 296 | |
| 297 | #if defined(__AVX2__1) |
| 298 | float iscale[4]; |
| 299 | __m256 srcv[4][32]; |
| 300 | __m256 iscale_vec[4]; |
| 301 | |
| 302 | for (int i = 0; i < nb; i++) { |
| 303 | for (int row_iter = 0; row_iter < 4; row_iter++) { |
| 304 | // Load elements into 4 AVX vectors |
| 305 | __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 ); |
| 306 | __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 8 ); |
| 307 | __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 16 ); |
| 308 | __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + 24 ); |
| 309 | |
| 310 | // Compute max(abs(e)) for the block |
| 311 | const __m256 signBit = _mm256_set1_ps( -0.0f ); |
| 312 | __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); |
| 313 | __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); |
| 314 | __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); |
| 315 | __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); |
| 316 | |
| 317 | __m256 maxAbs = _mm256_max_ps( abs0, abs1 ); |
| 318 | maxAbs = _mm256_max_ps( maxAbs, abs2 ); |
| 319 | maxAbs = _mm256_max_ps( maxAbs, abs3 ); |
| 320 | |
| 321 | __m256 mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v0), (0x00))); |
| 322 | __m256 mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v1), (0x00))); |
| 323 | __m256 mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v2), (0x00))); |
| 324 | __m256 mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v3), (0x00))); |
| 325 | |
| 326 | __m256 maskAbs = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); |
| 327 | |
| 328 | srcv[row_iter][0] = v0; |
| 329 | srcv[row_iter][1] = v1; |
| 330 | srcv[row_iter][2] = v2; |
| 331 | srcv[row_iter][3] = v3; |
| 332 | |
| 333 | for (int sb = 1; sb < 8; sb++) { |
| 334 | // Temporarily stores absolute quant values |
| 335 | __m256 tempAbs = maxAbs; |
| 336 | |
| 337 | // Load elements into 4 AVX vectors |
| 338 | __m256 v0 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32); |
| 339 | __m256 v1 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 8 ); |
| 340 | __m256 v2 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 16 ); |
| 341 | __m256 v3 = _mm256_loadu_ps( x + row_iter * k + i * 256 + sb * 32 + 24 ); |
| 342 | |
| 343 | // Compute max(abs(e)) for the block |
| 344 | __m256 abs0 = _mm256_andnot_ps( signBit, v0 ); |
| 345 | __m256 abs1 = _mm256_andnot_ps( signBit, v1 ); |
| 346 | __m256 abs2 = _mm256_andnot_ps( signBit, v2 ); |
| 347 | __m256 abs3 = _mm256_andnot_ps( signBit, v3 ); |
| 348 | |
| 349 | maxAbs = _mm256_max_ps( maxAbs, abs0 ); |
| 350 | maxAbs = _mm256_max_ps( maxAbs, abs1 ); |
| 351 | maxAbs = _mm256_max_ps( maxAbs, abs2 ); |
| 352 | maxAbs = _mm256_max_ps( maxAbs, abs3 ); |
| 353 | |
| 354 | __m256 mask_prev = _mm256_cmp_ps( tempAbs, maxAbs, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(tempAbs), ( __v8sf)(__m256)(maxAbs), (0x00))); |
| 355 | maskAbs = _mm256_and_ps( maskAbs, mask_prev ); |
| 356 | |
| 357 | mask0 = _mm256_cmp_ps( maxAbs, v0, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v0), (0x00))); |
| 358 | mask1 = _mm256_cmp_ps( maxAbs, v1, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v1), (0x00))); |
| 359 | mask2 = _mm256_cmp_ps( maxAbs, v2, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v2), (0x00))); |
| 360 | mask3 = _mm256_cmp_ps( maxAbs, v3, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxAbs), (__v8sf )(__m256)(v3), (0x00))); |
| 361 | |
| 362 | __m256 mask_curr = _mm256_or_ps(_mm256_or_ps(mask0, mask1),_mm256_or_ps(mask2, mask3)); |
| 363 | maskAbs = _mm256_or_ps(maskAbs, mask_curr); |
| 364 | |
| 365 | srcv[row_iter][sb * 4] = v0; |
| 366 | srcv[row_iter][sb * 4 + 1] = v1; |
| 367 | srcv[row_iter][sb * 4 + 2] = v2; |
| 368 | srcv[row_iter][sb * 4 + 3] = v3; |
| 369 | } |
| 370 | |
| 371 | __m128 max4 = _mm_max_ps( _mm256_extractf128_ps( maxAbs, 1 )((__m128)__builtin_ia32_vextractf128_ps256((__v8sf)(__m256)(maxAbs ), (int)(1))), _mm256_castps256_ps128( maxAbs ) ); |
| 372 | max4 = _mm_max_ps( max4, _mm_movehl_ps( max4, max4 ) ); |
| 373 | max4 = _mm_max_ss( max4, _mm_movehdup_ps( max4 ) ); |
| 374 | const float maxScalar = _mm_cvtss_f32( max4 ); |
| 375 | |
| 376 | __m256 maxScalarVec = _mm256_set1_ps(maxScalar); |
| 377 | |
| 378 | __m256 mask_next = _mm256_cmp_ps( maxScalarVec, maxAbs, _CMP_EQ_OQ )((__m256)__builtin_ia32_cmpps256((__v8sf)(__m256)(maxScalarVec ), (__v8sf)(__m256)(maxAbs), (0x00))); |
| 379 | __m256 finalMask = _mm256_and_ps(maskAbs, mask_next); |
| 380 | |
| 381 | const int mask = _mm256_movemask_ps(finalMask); |
| 382 | iscale[row_iter] = ( maxScalar != 0.0f ) ? 127.f / maxScalar : 0.0f; |
| 383 | |
| 384 | if(mask) { |
| 385 | iscale[row_iter] = ( maxScalar != 0.0f ) ? -127.f / maxScalar: 0.0f; |
| 386 | } |
| 387 | |
| 388 | y[i].d[row_iter] = maxScalar ? 1/iscale[row_iter] : 0; |
| 389 | iscale_vec[row_iter] = _mm256_set1_ps(iscale[row_iter]); |
| 390 | } |
| 391 | |
| 392 | __m256i quants_interleaved[32]; |
| 393 | for (int j = 0; j < 32; j++) { |
| 394 | // Apply the multiplier |
| 395 | __m256 v0 = _mm256_mul_ps(srcv[0][j], iscale_vec[0]); |
| 396 | __m256 v1 = _mm256_mul_ps(srcv[1][j], iscale_vec[1]); |
| 397 | __m256 v2 = _mm256_mul_ps(srcv[2][j], iscale_vec[2]); |
| 398 | __m256 v3 = _mm256_mul_ps(srcv[3][j], iscale_vec[3]); |
| 399 | |
| 400 | // Round to nearest integer |
| 401 | v0 = _mm256_round_ps( v0, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v0), ((0x0000U )))); |
| 402 | v1 = _mm256_round_ps( v1, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v1), ((0x0000U )))); |
| 403 | v2 = _mm256_round_ps( v2, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v2), ((0x0000U )))); |
| 404 | v3 = _mm256_round_ps( v3, _MM_ROUND_NEAREST )((__m256)__builtin_ia32_roundps256((__v8sf)(__m256)(v3), ((0x0000U )))); |
| 405 | |
| 406 | // Convert floats to integers |
| 407 | __m256i i0 = _mm256_cvtps_epi32( v0 ); |
| 408 | __m256i i1 = _mm256_cvtps_epi32( v1 ); |
| 409 | __m256i i2 = _mm256_cvtps_epi32( v2 ); |
| 410 | __m256i i3 = _mm256_cvtps_epi32( v3 ); |
| 411 | |
| 412 | // Convert int32 to int16 |
| 413 | i0 = _mm256_packs_epi32( i0, i1 ); |
| 414 | i2 = _mm256_packs_epi32( i2, i3 ); |
| 415 | // Convert int16 to int8 |
| 416 | i0 = _mm256_packs_epi16( i0, i2 ); |
| 417 | |
| 418 | // Permute and store the quantized weights in the required order after the pack instruction |
| 419 | const __m256i perm = _mm256_setr_epi32( 0, 4, 1, 5, 2, 6, 3, 7 ); |
| 420 | i0 = _mm256_permutevar8x32_epi32( i0, perm ); |
| 421 | |
| 422 | _mm256_storeu_si256((__m256i *)(y[i].qs + 32 * j), i0); |
| 423 | quants_interleaved[j] = i0; |
| 424 | } |
| 425 | |
| 426 | // Masks to shuffle the quants of corresponding sub blocks for rearranging quants for vectorized bsums computation |
| 427 | __m256i shuffle_mask_sb2 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 0, 1, 4, 5, 6, 7, 8, 9, 8, 9, 12, 13, 14, 15)); |
| 428 | shuffle_mask_sb2 = _mm256_permute2f128_si256(shuffle_mask_sb2, shuffle_mask_sb2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(shuffle_mask_sb2 ), (__v8si)(__m256i)(shuffle_mask_sb2), (int)(0))); |
| 429 | __m256i shuffle_mask_sb3 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 8, 9, 14, 15)); |
| 430 | shuffle_mask_sb3 = _mm256_permute2f128_si256(shuffle_mask_sb3, shuffle_mask_sb3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(shuffle_mask_sb3 ), (__v8si)(__m256i)(shuffle_mask_sb3), (int)(0))); |
| 431 | __m256i shuffle_mask_sb4 = _mm256_castsi128_si256(_mm_setr_epi8(0, 1, 2, 3, 4, 5, 0, 1, 8, 9, 10, 11, 12, 13, 8, 9)); |
| 432 | shuffle_mask_sb4 = _mm256_permute2f128_si256(shuffle_mask_sb4, shuffle_mask_sb4, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(shuffle_mask_sb4 ), (__v8si)(__m256i)(shuffle_mask_sb4), (int)(0))); |
| 433 | |
| 434 | for (int k = 0; k < 4; k++) { |
| 435 | // Quants from four different sub blocks are taken |
| 436 | __m256i q0 = quants_interleaved[k * 8 + 0]; |
| 437 | __m256i q1 = quants_interleaved[k * 8 + 1]; |
| 438 | __m256i q2 = quants_interleaved[k * 8 + 2]; |
| 439 | __m256i q3 = quants_interleaved[k * 8 + 3]; |
| 440 | __m256i q4 = quants_interleaved[k * 8 + 4]; |
| 441 | __m256i q5 = quants_interleaved[k * 8 + 5]; |
| 442 | __m256i q6 = quants_interleaved[k * 8 + 6]; |
| 443 | __m256i q7 = quants_interleaved[k * 8 + 7]; |
| 444 | |
| 445 | |
| 446 | // The below code block has the first half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time |
| 447 | __m256i sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); |
| 448 | __m256i sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(q0), ( __v16hi)(__m256i)(sb2_h1_shuffled), (int)(34))); |
| 449 | __m256i sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); |
| 450 | sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h1_interleaved ), (__v16hi)(__m256i)(sb3_h1_shuffled), (int)(68))); |
| 451 | __m256i sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); |
| 452 | sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h1_interleaved ), (__v16hi)(__m256i)(sb4_h1_shuffled), (int)(136))); |
| 453 | |
| 454 | __m256i one = _mm256_set1_epi8(1); |
| 455 | __m256i bsums_r1 = _mm256_maddubs_epi16(one, sb_h1_interleaved); |
| 456 | |
| 457 | for (int l = 0; l < 3; l++) { |
| 458 | // Quants value shifted to process next two values from each sub block |
| 459 | q0 = _mm256_srli_epi64(q0, 16); |
| 460 | q2 = _mm256_srli_epi64(q2, 16); |
| 461 | q4 = _mm256_srli_epi64(q4, 16); |
| 462 | q6 = _mm256_srli_epi64(q6, 16); |
| 463 | |
| 464 | sb2_h1_shuffled = _mm256_shuffle_epi8(q2, shuffle_mask_sb2); |
| 465 | sb_h1_interleaved = _mm256_blend_epi16(q0, sb2_h1_shuffled, 34)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(q0), ( __v16hi)(__m256i)(sb2_h1_shuffled), (int)(34))); |
| 466 | sb3_h1_shuffled = _mm256_shuffle_epi8(q4, shuffle_mask_sb3); |
| 467 | sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb3_h1_shuffled, 68)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h1_interleaved ), (__v16hi)(__m256i)(sb3_h1_shuffled), (int)(68))); |
| 468 | sb4_h1_shuffled = _mm256_shuffle_epi8(q6, shuffle_mask_sb4); |
| 469 | sb_h1_interleaved = _mm256_blend_epi16(sb_h1_interleaved, sb4_h1_shuffled, 136)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h1_interleaved ), (__v16hi)(__m256i)(sb4_h1_shuffled), (int)(136))); |
| 470 | |
| 471 | bsums_r1 = _mm256_add_epi16(bsums_r1, _mm256_maddubs_epi16(one, sb_h1_interleaved)); |
| 472 | } |
| 473 | |
| 474 | // The below code block has the second half of different sub blocks shuffled and blended so as to process 2 values from each sub block at a time |
| 475 | __m256i sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); |
| 476 | __m256i sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(q1), ( __v16hi)(__m256i)(sb2_h2_shuffled), (int)(34))); |
| 477 | __m256i sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); |
| 478 | sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h2_interleaved ), (__v16hi)(__m256i)(sb3_h2_shuffled), (int)(68))); |
| 479 | __m256i sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); |
| 480 | sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h2_interleaved ), (__v16hi)(__m256i)(sb4_h2_shuffled), (int)(136))); |
| 481 | |
| 482 | __m256i bsums_r2 = _mm256_maddubs_epi16(one, sb_h2_interleaved); |
| 483 | |
| 484 | for (int l = 0; l < 3; l++) { |
| 485 | // Quants value shifted to process next two values from each sub block |
| 486 | q1 = _mm256_srli_epi64(q1, 16); |
| 487 | q3 = _mm256_srli_epi64(q3, 16); |
| 488 | q5 = _mm256_srli_epi64(q5, 16); |
| 489 | q7 = _mm256_srli_epi64(q7, 16); |
| 490 | |
| 491 | sb2_h2_shuffled = _mm256_shuffle_epi8(q3, shuffle_mask_sb2); |
| 492 | sb_h2_interleaved = _mm256_blend_epi16(q1, sb2_h2_shuffled, 34)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(q1), ( __v16hi)(__m256i)(sb2_h2_shuffled), (int)(34))); |
| 493 | sb3_h2_shuffled = _mm256_shuffle_epi8(q5, shuffle_mask_sb3); |
| 494 | sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb3_h2_shuffled, 68)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h2_interleaved ), (__v16hi)(__m256i)(sb3_h2_shuffled), (int)(68))); |
| 495 | sb4_h2_shuffled = _mm256_shuffle_epi8(q7, shuffle_mask_sb4); |
| 496 | sb_h2_interleaved = _mm256_blend_epi16(sb_h2_interleaved, sb4_h2_shuffled, 136)((__m256i)__builtin_ia32_pblendw256((__v16hi)(__m256i)(sb_h2_interleaved ), (__v16hi)(__m256i)(sb4_h2_shuffled), (int)(136))); |
| 497 | |
| 498 | bsums_r2 = _mm256_add_epi16(bsums_r2, _mm256_maddubs_epi16(one, sb_h2_interleaved)); |
| 499 | } |
| 500 | |
| 501 | // Overall bsums in interleaved fashion computed by adding results of both halves |
| 502 | __m256i bsums_r = _mm256_add_epi16(bsums_r1, bsums_r2); |
| 503 | _mm256_storeu_si256((__m256i *)(y[i].bsums + 16 * k), bsums_r); |
| 504 | } |
| 505 | } |
| 506 | |
| 507 | #else |
| 508 | UNUSED(nb)(void)(nb); |
| 509 | UNUSED(y)(void)(y); |
| 510 | ggml_quantize_mat_q8_K_4x8_generic(x, vy, k); |
| 511 | #endif |
| 512 | } |
| 513 | |
| 514 | // |
| 515 | // GEMV/GEMM templates |
| 516 | // |
| 517 | |
| 518 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 519 | |
| 520 | // GEMV for 8x blocks of 32 4-bit quants with a single scale factor per block |
| 521 | template<typename block_tx8> |
| 522 | static void gemv_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc, __m256i signextendlut) { |
| 523 | static_assert( |
| 524 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 525 | std::is_same_v<block_tx8, block_iq4_nlx8> || |
| 526 | std::is_same_v<block_tx8, block_mxfp4x8>, |
| 527 | "Unsupported block type"); |
| 528 | |
| 529 | const int qk = QK8_032; |
| 530 | const int nb = n / qk; |
| 531 | |
| 532 | UNUSED(bs)(void)(bs); |
| 533 | |
| 534 | __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); |
| 535 | |
| 536 | // Permute mask used for easier vector processing at later stages |
| 537 | const __m256i m4b = _mm256_set1_epi8(0x0F); |
| 538 | |
| 539 | int64_t b_nb = n / 32; |
| 540 | |
| 541 | const block_tx8 * b_ptr_start = (const block_tx8 *)vx; |
| 542 | const block_q8_0 * a_ptr_start = (const block_q8_0 *)vy; |
| 543 | |
| 544 | // Process Q8_0 blocks one by one |
| 545 | for (int64_t y = 0; y < nr; y++) { |
| 546 | |
| 547 | // Pointers to LHS blocks of block_q8_0 format |
| 548 | const block_q8_0 * a_ptr = a_ptr_start + (y * nb); |
| 549 | |
| 550 | // Take group of eight blocks at each pass of the loop and perform dot product operation |
| 551 | for (int64_t x = 0; x < nc / 8; x++) { |
| 552 | |
| 553 | // Pointers to RHS blocks |
| 554 | const block_tx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 555 | |
| 556 | // Master FP accumulator |
| 557 | __m256 acc_row = _mm256_setzero_ps(); |
| 558 | |
| 559 | for (int64_t b = 0; b < nb; b++) { |
| 560 | // Load 8 blocks of 32 interleaved as 8 bytes (B0 - B7) |
| 561 | const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs)); |
| 562 | const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 1); |
| 563 | const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 2); |
| 564 | const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs) + 3); |
| 565 | |
| 566 | // 4-bit -> 8-bit - Sign is maintained |
| 567 | const __m256i rhs_vec_0123_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_0, m4b)); // B0(0-7) B1(0-7) B2(0-7) B3(0-7) |
| 568 | const __m256i rhs_vec_4567_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_0, m4b)); // B4(0-7) B5(0-7) B6(0-7) B7(0-7) |
| 569 | const __m256i rhs_vec_0123_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_0123_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15) |
| 570 | const __m256i rhs_vec_4567_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_vec_4567_1, m4b)); // B0(8-15) B1(8-15) B2(8-15) B3(8-15) |
| 571 | |
| 572 | const __m256i rhs_vec_0123_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b)); // B0(16-23) B1(16-23) B2(16-23) B3(16-23) |
| 573 | const __m256i rhs_vec_4567_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b)); // B4(16-23) B5(16-23) B6(16-23) B7(16-23) |
| 574 | const __m256i rhs_vec_0123_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b)); // B0(24-31) B1(24-31) B2(24-31) B3(24-31) |
| 575 | const __m256i rhs_vec_4567_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b)); // B4(24-31) B5(24-31) B6(24-31) B7(24-31) |
| 576 | |
| 577 | // Load the scale values for the 8 blocks interleaved in block_tx8 |
| 578 | __m256 col_scale_f32; |
| 579 | if constexpr ( |
| 580 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 581 | std::is_same_v<block_tx8, block_iq4_nlx8>) { |
| 582 | const __m128i changemask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0); |
| 583 | col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, changemask)_mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) b_ptr[b].d), changemask)); |
| 584 | } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) { |
| 585 | // Load 8 E8M0 exponents and convert to float via LUT |
| 586 | // Rearranged to match changemask order: 0,4,1,5,2,6,3,7 |
| 587 | col_scale_f32 = _mm256_set_ps( |
| 588 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[7])], |
| 589 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[3])], |
| 590 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[6])], |
| 591 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[2])], |
| 592 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[5])], |
| 593 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[1])], |
| 594 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[4])], |
| 595 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[0])]); |
| 596 | } |
| 597 | |
| 598 | // Load and convert to FP32 scale from block_q8_0 |
| 599 | const __m256 row_scale_f32 = _mm256_set1_ps(GGML_CPU_FP16_TO_FP32(a_ptr[b].d)ggml_lookup_fp16_to_fp32(a_ptr[b].d)); |
| 600 | |
| 601 | // Load the block values in block_q8_0 in batches of 16 bytes and replicate the same across 256 bit vector |
| 602 | __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)a_ptr[b].qs)); |
| 603 | __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16))); |
| 604 | |
| 605 | lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_0 ), (__v8si)(__m256i)(lhs_vec_0), (int)(0))); // A0 (0-15) A0(0-15) |
| 606 | lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_1 ), (__v8si)(__m256i)(lhs_vec_1), (int)(0))); // A0 (16-31) A0(16-31)) |
| 607 | |
| 608 | __m256i iacc = _mm256_setzero_si256(); |
| 609 | |
| 610 | // Dot product done within 32 bit lanes and accumulated in the same vector |
| 611 | // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3) |
| 612 | // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7) |
| 613 | // ........................................................................... |
| 614 | // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31) |
| 615 | |
| 616 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_0 ,_mm256_shuffle_epi32(rhs_vec_4567_0, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_0 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_0), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(0)))); |
| 617 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_0, 177) ,rhs_vec_4567_0, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_0), ( int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_0), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(85)))); |
| 618 | |
| 619 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_1 ,_mm256_shuffle_epi32(rhs_vec_4567_1, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_1 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_1), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(170)))); |
| 620 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_1, 177) ,rhs_vec_4567_1, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_1), ( int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_1), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(255)))); |
| 621 | |
| 622 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_2 ,_mm256_shuffle_epi32(rhs_vec_4567_2, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_2 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_2), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(0)))); |
| 623 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_2, 177) ,rhs_vec_4567_2, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_2), ( int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_2), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(85)))); |
| 624 | |
| 625 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(rhs_vec_0123_3 ,_mm256_shuffle_epi32(rhs_vec_4567_3, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_3 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_3), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(170)))); |
| 626 | iacc = mul_sum_i8_pairs_acc_int32x8(iacc, _mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_3, 177) ,rhs_vec_4567_3, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_3), ( int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_3), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(255)))); |
| 627 | |
| 628 | // Accumulated values multiplied with appropriate scales |
| 629 | acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row); |
| 630 | } |
| 631 | |
| 632 | // Accumulated output values permuted so as to be stored in appropriate order post accumulation |
| 633 | acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask); |
| 634 | _mm256_storeu_ps(s + (y * nr + x * 8), acc_row); |
| 635 | } |
| 636 | } |
| 637 | } |
| 638 | |
| 639 | // GEMM for 8x blocks of 32 4-bit quants with a single scale factor per block |
| 640 | template<typename block_tx8> |
| 641 | static void gemm_q4_b32_8x8_q8_0_lut_avx(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc, __m256i signextendlut) { |
| 642 | static_assert( |
| 643 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 644 | std::is_same_v<block_tx8, block_iq4_nlx8> || |
| 645 | std::is_same_v<block_tx8, block_mxfp4x8>, |
| 646 | "Unsupported block type"); |
| 647 | |
| 648 | const int qk = QK8_032; |
| 649 | const int nb = n / qk; |
| 650 | |
| 651 | const block_tx8 * b_ptr_start = (const block_tx8 *)vx; |
| 652 | const block_q8_0x4 * a_ptr_start = (const block_q8_0x4 *)vy; |
| 653 | |
| 654 | int64_t b_nb = n / 32; |
| 655 | int64_t y = 0; |
| 656 | // Mask to mask out nibbles from packed bytes |
| 657 | const __m256i m4b = _mm256_set1_epi8(0x0F); |
| 658 | const __m128i loadMask = _mm_blend_epi32(_mm_setzero_si128(), _mm_set1_epi32(0xFFFFFFFF), 3)((__m128i)__builtin_ia32_pblendd128((__v4si)(__m128i)(_mm_setzero_si128 ()), (__v4si)(__m128i)(_mm_set1_epi32(0xFFFFFFFF)), (int)(3)) ); |
| 659 | // Permute mask used for easier vector processing at later stages |
| 660 | __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4); |
| 661 | int64_t xstart = 0; |
| 662 | int anr = nr - nr%16; // Used to align nr with boundary of 16 |
| 663 | #if defined(__AVX512BW__) && defined(__AVX512DQ__) |
| 664 | int anc = nc - nc%16; // Used to align nc with boundary of 16 |
| 665 | // Mask to mask out nibbles from packed bytes expanded to 512 bit length |
| 666 | const __m512i m4bexpanded = _mm512_set1_epi8(0x0F); |
| 667 | // Lookup table to convert signed nibbles to signed bytes expanded to 512 bit length |
| 668 | __m512i signextendlutexpanded = _mm512_inserti32x8(_mm512_castsi256_si512(signextendlut), signextendlut, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (signextendlut)), (__v8si)(__m256i)(signextendlut), (int)(1)) ); |
| 669 | |
| 670 | // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation |
| 671 | for (; y < anr / 4; y += 4) { |
| 672 | |
| 673 | const block_q8_0x4 * a_ptrs[4]; |
| 674 | |
| 675 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 676 | for (int i = 0; i < 3; ++i) { |
| 677 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 678 | } |
| 679 | |
| 680 | // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation |
| 681 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 682 | |
| 683 | const block_tx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 684 | const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 685 | |
| 686 | // Master FP accumulators |
| 687 | __m512 acc_rows[16]; |
| 688 | for (int i = 0; i < 16; i++) { |
| 689 | acc_rows[i] = _mm512_setzero_ps(); |
| 690 | } |
| 691 | |
| 692 | for (int64_t b = 0; b < nb; b++) { |
| 693 | // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF |
| 694 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs)); |
| 695 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32)); |
| 696 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64)); |
| 697 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96)); |
| 698 | |
| 699 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs)); |
| 700 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32)); |
| 701 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64)); |
| 702 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96)); |
| 703 | |
| 704 | // Save the values in the following vectors in the formats B0B1B4B5B8B9BCBD, B2B3B6B7BABBBEBF for further processing and storing of values |
| 705 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 706 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 707 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 708 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 709 | |
| 710 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 711 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 712 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 713 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 714 | |
| 715 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 716 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 717 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 718 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 719 | |
| 720 | // 4-bit -> 8-bit - Sign is maintained |
| 721 | const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7) |
| 722 | const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7) |
| 723 | |
| 724 | const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15) |
| 725 | const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15) |
| 726 | |
| 727 | const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23) |
| 728 | const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23) |
| 729 | |
| 730 | const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31) |
| 731 | const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31) |
| 732 | |
| 733 | // Shuffle pattern one - right side input |
| 734 | const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_0 ), (int)((_MM_PERM_ENUM)136))); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3) |
| 735 | const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_0 ), (int)((_MM_PERM_ENUM)136))); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3) |
| 736 | |
| 737 | const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_1 ), (int)((_MM_PERM_ENUM)136))); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11) |
| 738 | const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_1 ), (int)((_MM_PERM_ENUM)136))); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11) |
| 739 | |
| 740 | const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_2 ), (int)((_MM_PERM_ENUM)136))); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19) |
| 741 | const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_2 ), (int)((_MM_PERM_ENUM)136))); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19) |
| 742 | |
| 743 | const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_3 ), (int)((_MM_PERM_ENUM)136))); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27) |
| 744 | const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_3 ), (int)((_MM_PERM_ENUM)136))); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27) |
| 745 | |
| 746 | // Shuffle pattern two - right side input |
| 747 | |
| 748 | const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_0 ), (int)((_MM_PERM_ENUM)221))); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7) |
| 749 | const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_0 ), (int)((_MM_PERM_ENUM)221))); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7) |
| 750 | |
| 751 | const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_1 ), (int)((_MM_PERM_ENUM)221))); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15) |
| 752 | const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_1 ), (int)((_MM_PERM_ENUM)221))); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15) |
| 753 | |
| 754 | const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_2 ), (int)((_MM_PERM_ENUM)221))); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23) |
| 755 | const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_2 ), (int)((_MM_PERM_ENUM)221))); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23) |
| 756 | |
| 757 | const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_3 ), (int)((_MM_PERM_ENUM)221))); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31) |
| 758 | const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_3 ), (int)((_MM_PERM_ENUM)221))); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31) |
| 759 | |
| 760 | // Scale values - Load the weight scale values of two block_tx8 |
| 761 | __m512 col_scale_f32; |
| 762 | if constexpr ( |
| 763 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 764 | std::is_same_v<block_tx8, block_iq4_nlx8>) { |
| 765 | col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 766 | } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) { |
| 767 | //TODO: simd-ify |
| 768 | col_scale_f32 = _mm512_set_ps( |
| 769 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[7])], |
| 770 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[6])], |
| 771 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[5])], |
| 772 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[4])], |
| 773 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[3])], |
| 774 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[2])], |
| 775 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[1])], |
| 776 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[0])], |
| 777 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[7])], |
| 778 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[6])], |
| 779 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[5])], |
| 780 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[4])], |
| 781 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[3])], |
| 782 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[2])], |
| 783 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[1])], |
| 784 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[0])]); |
| 785 | } |
| 786 | |
| 787 | // Process LHS in pairs of rows |
| 788 | for (int rp = 0; rp < 4; rp++) { |
| 789 | |
| 790 | // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3 |
| 791 | // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector |
| 792 | __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs))); |
| 793 | __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_0 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_0), (int)(0))); |
| 794 | __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_0 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_0), (int)(17))); |
| 795 | __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32))); |
| 796 | __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_1 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_1), (int)(0))); |
| 797 | __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_1 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_1), (int)(17))); |
| 798 | __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64))); |
| 799 | __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_2 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_2), (int)(0))); |
| 800 | __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_2 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_2), (int)(17))); |
| 801 | __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96))); |
| 802 | __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_3 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_3), (int)(0))); |
| 803 | __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_3 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_3), (int)(17))); |
| 804 | |
| 805 | __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_0)), (__v8si)(__m256i)(lhs_mat_ymm_01_0), (int )(1))); |
| 806 | __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_0)), (__v8si)(__m256i)(lhs_mat_ymm_23_0), (int )(1))); |
| 807 | __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_1)), (__v8si)(__m256i)(lhs_mat_ymm_01_1), (int )(1))); |
| 808 | __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_1)), (__v8si)(__m256i)(lhs_mat_ymm_23_1), (int )(1))); |
| 809 | __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_2)), (__v8si)(__m256i)(lhs_mat_ymm_01_2), (int )(1))); |
| 810 | __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_2)), (__v8si)(__m256i)(lhs_mat_ymm_23_2), (int )(1))); |
| 811 | __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_3)), (__v8si)(__m256i)(lhs_mat_ymm_01_3), (int )(1))); |
| 812 | __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_3)), (__v8si)(__m256i)(lhs_mat_ymm_23_3), (int )(1))); |
| 813 | |
| 814 | // Shuffle pattern one - left side input |
| 815 | |
| 816 | const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_0 ), (int)((_MM_PERM_ENUM)160))); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) |
| 817 | const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_0 ), (int)((_MM_PERM_ENUM)160))); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) |
| 818 | |
| 819 | const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_1 ), (int)((_MM_PERM_ENUM)160))); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) |
| 820 | const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_1 ), (int)((_MM_PERM_ENUM)160))); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) |
| 821 | |
| 822 | const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_2 ), (int)((_MM_PERM_ENUM)160))); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) |
| 823 | const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_2 ), (int)((_MM_PERM_ENUM)160))); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) |
| 824 | |
| 825 | const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_3 ), (int)((_MM_PERM_ENUM)160))); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) |
| 826 | const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_3 ), (int)((_MM_PERM_ENUM)160))); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) |
| 827 | |
| 828 | // Shuffle pattern two - left side input |
| 829 | |
| 830 | const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_0 ), (int)((_MM_PERM_ENUM)245))); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) |
| 831 | const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_0 ), (int)((_MM_PERM_ENUM)245))); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) |
| 832 | |
| 833 | const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_1 ), (int)((_MM_PERM_ENUM)245))); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) |
| 834 | const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_1 ), (int)((_MM_PERM_ENUM)245))); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) |
| 835 | |
| 836 | const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_2 ), (int)((_MM_PERM_ENUM)245))); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) |
| 837 | const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_2 ), (int)((_MM_PERM_ENUM)245))); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) |
| 838 | |
| 839 | const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_3 ), (int)((_MM_PERM_ENUM)245))); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) |
| 840 | const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_3 ), (int)((_MM_PERM_ENUM)245))); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) |
| 841 | |
| 842 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 843 | // Resembles MMLAs into 2x2 matrices in ARM Version |
| 844 | const __m512i zero = _mm512_setzero_epi32_mm512_setzero_si512(); |
| 845 | __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1); |
| 846 | __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1); |
| 847 | __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1); |
| 848 | __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1); |
| 849 | __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2); |
| 850 | __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2); |
| 851 | __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2); |
| 852 | __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2); |
| 853 | |
| 854 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 855 | __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2); |
| 856 | __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2); |
| 857 | __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2); |
| 858 | __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2); |
| 859 | |
| 860 | |
| 861 | // Straighten out to make 4 row vectors |
| 862 | __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01 ), (int)((_MM_PERM_ENUM)78)))); |
| 863 | __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01); |
| 864 | __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11 ), (int)((_MM_PERM_ENUM)78)))); |
| 865 | __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11); |
| 866 | |
| 867 | // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes |
| 868 | const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptrs[rp][b].d), loadMask), 68)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(_mm_maskload_epi32 ((int const*)(a_ptrs[rp][b].d), loadMask)), (int)(68))); |
| 869 | const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16); |
| 870 | |
| 871 | // Multiply with appropriate scales and accumulate |
| 872 | acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 873 | acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 874 | acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 875 | acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 876 | } |
| 877 | } |
| 878 | |
| 879 | // Store the accumulated values |
| 880 | for (int i = 0; i < 16; i++) { |
| 881 | _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]); |
| 882 | } |
| 883 | } |
| 884 | } |
| 885 | |
| 886 | // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation |
| 887 | for (; y < nr / 4; y ++) { |
| 888 | const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb); |
| 889 | |
| 890 | // Take group of two block_tx8 structures at each pass of the loop and perform dot product operation |
| 891 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 892 | |
| 893 | const block_tx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 894 | const block_tx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 895 | |
| 896 | // Master FP accumulators |
| 897 | __m512 acc_rows[4]; |
| 898 | for (int i = 0; i < 4; i++) { |
| 899 | acc_rows[i] = _mm512_setzero_ps(); |
| 900 | } |
| 901 | |
| 902 | for (int64_t b = 0; b < nb; b++) { |
| 903 | // Load the sixteen blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....BE,BF |
| 904 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs)); |
| 905 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 32)); |
| 906 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 64)); |
| 907 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_0[b].qs + 96)); |
| 908 | |
| 909 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs)); |
| 910 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 32)); |
| 911 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 64)); |
| 912 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i *)(b_ptr_1[b].qs + 96)); |
| 913 | |
| 914 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 915 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 916 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 917 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 918 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 919 | |
| 920 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 921 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 922 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 923 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 924 | |
| 925 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 926 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 927 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 928 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 929 | |
| 930 | // 4-bit -> 8-bit - Sign is maintained |
| 931 | const __m512i rhs_mat_014589CD_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) B8(0-7) B9(0-7) BC(0-7) BD(0-7) |
| 932 | const __m512i rhs_mat_2367ABEF_0 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) BA(0-7) BB(0-7) BE(0-7) BF(0-7) |
| 933 | |
| 934 | const __m512i rhs_mat_014589CD_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) B8(8-15) B9(8-15) BC(8-15) BD(8-15) |
| 935 | const __m512i rhs_mat_2367ABEF_1 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) BA(8-15) BB(8-15) BE(8-15) BF(8-15) |
| 936 | |
| 937 | const __m512i rhs_mat_014589CD_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) B8(16-23) B9(16-23) BC(16-23) BD(16-23) |
| 938 | const __m512i rhs_mat_2367ABEF_2 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) BA(16-23) BB(16-23) BE(16-23) BF(16-23) |
| 939 | |
| 940 | const __m512i rhs_mat_014589CD_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) B8(24-31) B9(24-31) BC(24-31) BD(24-31) |
| 941 | const __m512i rhs_mat_2367ABEF_3 = _mm512_shuffle_epi8(signextendlutexpanded, _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) BA(24-31) BB(24-31) BE(24-31) BF(24-31) |
| 942 | |
| 943 | // Shuffle pattern one - right side input |
| 944 | const __m512i rhs_mat_014589CD_0_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_0 ), (int)((_MM_PERM_ENUM)136))); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) B8(0-3) B9(0-3) B8(0-3) B9(0-3) BC(0-3) BD(0-3) BC(0-3) BD(0-3) |
| 945 | const __m512i rhs_mat_2367ABEF_0_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_0 ), (int)((_MM_PERM_ENUM)136))); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) BA(0-3) BB(0-3) BA(0-3) BB(0-3) BE(0-3) BF(0-3) BE(0-3) BF(0-3) |
| 946 | |
| 947 | const __m512i rhs_mat_014589CD_1_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_1 ), (int)((_MM_PERM_ENUM)136))); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) B8(8-11) B9(8-11) B8(8-11) B9(8-11) BC(8-11) BD(8-11) BC(8-11) BD(8-11) |
| 948 | const __m512i rhs_mat_2367ABEF_1_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_1 ), (int)((_MM_PERM_ENUM)136))); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) BA(8-11) BB(8-11) BA(8-11) BB(8-11) BE(8-11) BF(8-11) BE(8-11) BF(8-11) |
| 949 | |
| 950 | const __m512i rhs_mat_014589CD_2_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_2 ), (int)((_MM_PERM_ENUM)136))); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) B8(16-19) B9(16-19) B8(16-19) B9(16-19) BC(16-19) BD(16-19) BC(16-19) BD(16-19) |
| 951 | const __m512i rhs_mat_2367ABEF_2_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_2 ), (int)((_MM_PERM_ENUM)136))); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) BA(16-19) BB(16-19) BA(16-19) BB(16-19) BE(16-19) BF(16-19) BE(16-19) BF(16-19) |
| 952 | |
| 953 | const __m512i rhs_mat_014589CD_3_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_3 ), (int)((_MM_PERM_ENUM)136))); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) B8(24-27) B9(24-27) B8(24-27) B9(24-27) BC(24-27) BD(24-27) BC(24-27) BD(24-27) |
| 954 | const __m512i rhs_mat_2367ABEF_3_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_3 ), (int)((_MM_PERM_ENUM)136))); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) BA(24-27) BB(24-27) BA(24-27) BB(24-27) BE(24-27) BF(24-27) BE(24-27) BF(24-27) |
| 955 | |
| 956 | // Shuffle pattern two - right side input |
| 957 | |
| 958 | const __m512i rhs_mat_014589CD_0_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_0, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_0 ), (int)((_MM_PERM_ENUM)221))); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) B8(4-7) B9(4-7) B8(4-7) B9(4-7) BC(4-7) BD(4-7) BC(4-7) BD(4-7) |
| 959 | const __m512i rhs_mat_2367ABEF_0_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_0, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_0 ), (int)((_MM_PERM_ENUM)221))); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) BA(4-7) BB(4-7) BA(4-7) BB(4-7) BE(4-7) BF(4-7) BE(4-7) BF(4-7) |
| 960 | |
| 961 | const __m512i rhs_mat_014589CD_1_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_1, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_1 ), (int)((_MM_PERM_ENUM)221))); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) B8(12-15) B9(12-15) B8(12-15) B9(12-15) BC(12-15) BD(12-15) BC(12-15) BD(12-15) |
| 962 | const __m512i rhs_mat_2367ABEF_1_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_1, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_1 ), (int)((_MM_PERM_ENUM)221))); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) BA(12-15) BB(12-15) BA(12-15) BB(12-15) BE(12-15) BF(12-15) BE(12-15) BF(12-15) |
| 963 | |
| 964 | const __m512i rhs_mat_014589CD_2_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_2, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_2 ), (int)((_MM_PERM_ENUM)221))); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) B8(20-23) B9(20-23) B8(20-23) B9(20-23) BC(20-23) BD(20-23) BC(20-23) BD(20-23) |
| 965 | const __m512i rhs_mat_2367ABEF_2_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_2, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_2 ), (int)((_MM_PERM_ENUM)221))); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) BA(20-23) BB(20-23) BA(20-23) BB(20-23) BE(20-23) BF(20-23) BE(20-23) BF(20-23) |
| 966 | |
| 967 | const __m512i rhs_mat_014589CD_3_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_3, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_3 ), (int)((_MM_PERM_ENUM)221))); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) B8(28-31) B9(28-31) B8(28-31) B9(28-31) BC(28-31) BD(28-31) BC(28-31) BD(28-31) |
| 968 | const __m512i rhs_mat_2367ABEF_3_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_3, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_3 ), (int)((_MM_PERM_ENUM)221))); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) BA(28-31) BB(28-31) BA(28-31) BB(28-31) BE(28-31) BF(28-31) BE(28-31) BF(28-31) |
| 969 | |
| 970 | |
| 971 | // Scale values - Load the weight scale values of two block_tx8 |
| 972 | __m512 col_scale_f32; |
| 973 | if constexpr ( |
| 974 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 975 | std::is_same_v<block_tx8, block_iq4_nlx8>) { |
| 976 | col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 977 | } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) { |
| 978 | //TODO: simd-ify |
| 979 | col_scale_f32 = _mm512_set_ps( |
| 980 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[7])], |
| 981 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[6])], |
| 982 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[5])], |
| 983 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[4])], |
| 984 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[3])], |
| 985 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[2])], |
| 986 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[1])], |
| 987 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_1[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_1[b].e[0])], |
| 988 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[7])], |
| 989 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[6])], |
| 990 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[5])], |
| 991 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[4])], |
| 992 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[3])], |
| 993 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[2])], |
| 994 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[1])], |
| 995 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr_0[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr_0[b].e[0])]); |
| 996 | } |
| 997 | |
| 998 | // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3 |
| 999 | // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector |
| 1000 | __m256i lhs_mat_ymm_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs))); |
| 1001 | __m256i lhs_mat_ymm_01_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_0 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_0), (int)(0))); |
| 1002 | __m256i lhs_mat_ymm_23_0 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_0, lhs_mat_ymm_0123_0, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_0 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_0), (int)(17))); |
| 1003 | __m256i lhs_mat_ymm_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32))); |
| 1004 | __m256i lhs_mat_ymm_01_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_1 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_1), (int)(0))); |
| 1005 | __m256i lhs_mat_ymm_23_1 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_1, lhs_mat_ymm_0123_1, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_1 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_1), (int)(17))); |
| 1006 | __m256i lhs_mat_ymm_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64))); |
| 1007 | __m256i lhs_mat_ymm_01_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_2 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_2), (int)(0))); |
| 1008 | __m256i lhs_mat_ymm_23_2 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_2, lhs_mat_ymm_0123_2, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_2 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_2), (int)(17))); |
| 1009 | __m256i lhs_mat_ymm_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96))); |
| 1010 | __m256i lhs_mat_ymm_01_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_3 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_3), (int)(0))); |
| 1011 | __m256i lhs_mat_ymm_23_3 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_3, lhs_mat_ymm_0123_3, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_3 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_3), (int)(17))); |
| 1012 | |
| 1013 | __m512i lhs_mat_01_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_0), lhs_mat_ymm_01_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_0)), (__v8si)(__m256i)(lhs_mat_ymm_01_0), (int )(1))); |
| 1014 | __m512i lhs_mat_23_0 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_0), lhs_mat_ymm_23_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_0)), (__v8si)(__m256i)(lhs_mat_ymm_23_0), (int )(1))); |
| 1015 | __m512i lhs_mat_01_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_1), lhs_mat_ymm_01_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_1)), (__v8si)(__m256i)(lhs_mat_ymm_01_1), (int )(1))); |
| 1016 | __m512i lhs_mat_23_1 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_1), lhs_mat_ymm_23_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_1)), (__v8si)(__m256i)(lhs_mat_ymm_23_1), (int )(1))); |
| 1017 | __m512i lhs_mat_01_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_2), lhs_mat_ymm_01_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_2)), (__v8si)(__m256i)(lhs_mat_ymm_01_2), (int )(1))); |
| 1018 | __m512i lhs_mat_23_2 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_2), lhs_mat_ymm_23_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_2)), (__v8si)(__m256i)(lhs_mat_ymm_23_2), (int )(1))); |
| 1019 | __m512i lhs_mat_01_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_3), lhs_mat_ymm_01_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_3)), (__v8si)(__m256i)(lhs_mat_ymm_01_3), (int )(1))); |
| 1020 | __m512i lhs_mat_23_3 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_3), lhs_mat_ymm_23_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_3)), (__v8si)(__m256i)(lhs_mat_ymm_23_3), (int )(1))); |
| 1021 | |
| 1022 | // Shuffle pattern one - left side input |
| 1023 | |
| 1024 | const __m512i lhs_mat_01_0_sp1 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_0 ), (int)((_MM_PERM_ENUM)160))); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) |
| 1025 | const __m512i lhs_mat_23_0_sp1 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_0 ), (int)((_MM_PERM_ENUM)160))); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) |
| 1026 | |
| 1027 | const __m512i lhs_mat_01_1_sp1 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_1 ), (int)((_MM_PERM_ENUM)160))); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) |
| 1028 | const __m512i lhs_mat_23_1_sp1 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_1 ), (int)((_MM_PERM_ENUM)160))); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) |
| 1029 | |
| 1030 | const __m512i lhs_mat_01_2_sp1 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_2 ), (int)((_MM_PERM_ENUM)160))); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) |
| 1031 | const __m512i lhs_mat_23_2_sp1 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_2 ), (int)((_MM_PERM_ENUM)160))); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) |
| 1032 | |
| 1033 | const __m512i lhs_mat_01_3_sp1 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_3 ), (int)((_MM_PERM_ENUM)160))); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) |
| 1034 | const __m512i lhs_mat_23_3_sp1 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_3 ), (int)((_MM_PERM_ENUM)160))); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) |
| 1035 | |
| 1036 | // Shuffle pattern two - left side input |
| 1037 | |
| 1038 | const __m512i lhs_mat_01_0_sp2 = _mm512_shuffle_epi32(lhs_mat_01_0, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_0 ), (int)((_MM_PERM_ENUM)245))); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) |
| 1039 | const __m512i lhs_mat_23_0_sp2 = _mm512_shuffle_epi32(lhs_mat_23_0, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_0 ), (int)((_MM_PERM_ENUM)245))); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) |
| 1040 | |
| 1041 | const __m512i lhs_mat_01_1_sp2 = _mm512_shuffle_epi32(lhs_mat_01_1, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_1 ), (int)((_MM_PERM_ENUM)245))); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) |
| 1042 | const __m512i lhs_mat_23_1_sp2 = _mm512_shuffle_epi32(lhs_mat_23_1, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_1 ), (int)((_MM_PERM_ENUM)245))); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) |
| 1043 | |
| 1044 | const __m512i lhs_mat_01_2_sp2 = _mm512_shuffle_epi32(lhs_mat_01_2, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_2 ), (int)((_MM_PERM_ENUM)245))); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) |
| 1045 | const __m512i lhs_mat_23_2_sp2 = _mm512_shuffle_epi32(lhs_mat_23_2, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_2 ), (int)((_MM_PERM_ENUM)245))); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) |
| 1046 | |
| 1047 | const __m512i lhs_mat_01_3_sp2 = _mm512_shuffle_epi32(lhs_mat_01_3, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_3 ), (int)((_MM_PERM_ENUM)245))); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) |
| 1048 | const __m512i lhs_mat_23_3_sp2 = _mm512_shuffle_epi32(lhs_mat_23_3, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_3 ), (int)((_MM_PERM_ENUM)245))); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) |
| 1049 | |
| 1050 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 1051 | // Resembles MMLAs into 2x2 matrices in ARM Version |
| 1052 | const __m512i zero = _mm512_setzero_epi32_mm512_setzero_si512(); |
| 1053 | __m512i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_01_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_01_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_01_0_sp1, rhs_mat_014589CD_0_sp1); |
| 1054 | __m512i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367ABEF_0_sp1); |
| 1055 | __m512i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_014589CD_3_sp1), lhs_mat_23_2_sp1, rhs_mat_014589CD_2_sp1), lhs_mat_23_1_sp1, rhs_mat_014589CD_1_sp1), lhs_mat_23_0_sp1, rhs_mat_014589CD_0_sp1); |
| 1056 | __m512i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp1, rhs_mat_2367ABEF_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367ABEF_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367ABEF_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367ABEF_0_sp1); |
| 1057 | __m512i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_01_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_01_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_01_0_sp2, rhs_mat_014589CD_0_sp2); |
| 1058 | __m512i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_01_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367ABEF_0_sp2); |
| 1059 | __m512i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_014589CD_3_sp2), lhs_mat_23_2_sp2, rhs_mat_014589CD_2_sp2), lhs_mat_23_1_sp2, rhs_mat_014589CD_1_sp2), lhs_mat_23_0_sp2, rhs_mat_014589CD_0_sp2); |
| 1060 | __m512i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(mul_sum_i8_pairs_acc_int32x16(zero, lhs_mat_23_3_sp2, rhs_mat_2367ABEF_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367ABEF_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367ABEF_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367ABEF_0_sp2); |
| 1061 | |
| 1062 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 1063 | __m512i iacc_mat_00 = _mm512_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2); |
| 1064 | __m512i iacc_mat_01 = _mm512_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2); |
| 1065 | __m512i iacc_mat_10 = _mm512_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2); |
| 1066 | __m512i iacc_mat_11 = _mm512_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2); |
| 1067 | |
| 1068 | |
| 1069 | // Straighten out to make 4 row vectors |
| 1070 | __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01 ), (int)((_MM_PERM_ENUM)78)))); |
| 1071 | __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01); |
| 1072 | __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11 ), (int)((_MM_PERM_ENUM)78)))); |
| 1073 | __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11); |
| 1074 | |
| 1075 | // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes |
| 1076 | const __m128i row_scale_f16 = _mm_shuffle_epi32(_mm_maskload_epi32((int const*)(a_ptr[b].d), loadMask), 68)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(_mm_maskload_epi32 ((int const*)(a_ptr[b].d), loadMask)), (int)(68))); |
| 1077 | const __m512 row_scale_f32 = GGML_F32Cx16_REPEAT_LOAD(row_scale_f16); |
| 1078 | |
| 1079 | // Multiply with appropriate scales and accumulate |
| 1080 | acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 1081 | acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 1082 | acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 1083 | acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 1084 | } |
| 1085 | |
| 1086 | // Store the accumulated values |
| 1087 | for (int i = 0; i < 4; i++) { |
| 1088 | _mm512_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]); |
| 1089 | } |
| 1090 | } |
| 1091 | } |
| 1092 | if (anc != nc) { |
| 1093 | xstart = anc/8; |
| 1094 | y = 0; |
| 1095 | } |
| 1096 | #endif // __AVX512BW__ && __AVX512DQ__ |
| 1097 | |
| 1098 | // Take group of four block_q8_0x4 structures at each pass of the loop and perform dot product operation |
| 1099 | |
| 1100 | for (; y < anr / 4; y += 4) { |
| 1101 | const block_q8_0x4 * a_ptrs[4]; |
| 1102 | |
| 1103 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 1104 | for (int i = 0; i < 3; ++i) { |
| 1105 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 1106 | } |
| 1107 | |
| 1108 | // Take group of eight block_tx8 structures at each pass of the loop and perform dot product operation |
| 1109 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 1110 | |
| 1111 | const block_tx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 1112 | |
| 1113 | // Master FP accumulators |
| 1114 | __m256 acc_rows[16]; |
| 1115 | for (int i = 0; i < 16; i++) { |
| 1116 | acc_rows[i] = _mm256_setzero_ps(); |
| 1117 | } |
| 1118 | |
| 1119 | for (int64_t b = 0; b < nb; b++) { |
| 1120 | // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 |
| 1121 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs)); |
| 1122 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32)); |
| 1123 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64)); |
| 1124 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96)); |
| 1125 | |
| 1126 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 1127 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 1128 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 1129 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 1130 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 1131 | |
| 1132 | // 4-bit -> 8-bit - Sign is maintained |
| 1133 | const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) |
| 1134 | const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) |
| 1135 | |
| 1136 | const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) |
| 1137 | const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) |
| 1138 | |
| 1139 | const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) |
| 1140 | const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) |
| 1141 | |
| 1142 | const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) |
| 1143 | const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) |
| 1144 | |
| 1145 | // Shuffle pattern one - right side input |
| 1146 | const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_0 ), (int)(136))); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) |
| 1147 | const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_0 ), (int)(136))); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) |
| 1148 | |
| 1149 | const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_1 ), (int)(136))); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) |
| 1150 | const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_1 ), (int)(136))); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) |
| 1151 | |
| 1152 | const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_2 ), (int)(136))); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) |
| 1153 | const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_2 ), (int)(136))); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) |
| 1154 | |
| 1155 | const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_3 ), (int)(136))); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) |
| 1156 | const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_3 ), (int)(136))); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) |
| 1157 | |
| 1158 | // Shuffle pattern two - right side input |
| 1159 | |
| 1160 | const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_0 ), (int)(221))); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) |
| 1161 | const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_0 ), (int)(221))); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) |
| 1162 | |
| 1163 | const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_1 ), (int)(221))); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) |
| 1164 | const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_1 ), (int)(221))); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) |
| 1165 | |
| 1166 | const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_2 ), (int)(221))); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) |
| 1167 | const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_2 ), (int)(221))); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) |
| 1168 | |
| 1169 | const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_3 ), (int)(221))); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) |
| 1170 | const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_3 ), (int)(221))); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) |
| 1171 | |
| 1172 | // Scale values - Load the wight scale values of block_tx8 |
| 1173 | __m256 col_scale_f32; |
| 1174 | if constexpr ( |
| 1175 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 1176 | std::is_same_v<block_tx8, block_iq4_nlx8>) { |
| 1177 | col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 1178 | } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) { |
| 1179 | col_scale_f32 = _mm256_set_ps( |
| 1180 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[7])], |
| 1181 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[6])], |
| 1182 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[5])], |
| 1183 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[4])], |
| 1184 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[3])], |
| 1185 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[2])], |
| 1186 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[1])], |
| 1187 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[0])]); |
| 1188 | } |
| 1189 | |
| 1190 | // Process LHS in groups of four |
| 1191 | for (int rp = 0; rp < 4; rp++) { |
| 1192 | // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3 |
| 1193 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 1194 | __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs))); |
| 1195 | __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_0 ), (__v8si)(__m256i)(lhs_mat_0123_0), (int)(0))); |
| 1196 | __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_0 ), (__v8si)(__m256i)(lhs_mat_0123_0), (int)(17))); |
| 1197 | __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 32))); |
| 1198 | __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_1 ), (__v8si)(__m256i)(lhs_mat_0123_1), (int)(0))); |
| 1199 | __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_1 ), (__v8si)(__m256i)(lhs_mat_0123_1), (int)(17))); |
| 1200 | __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 64))); |
| 1201 | __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_2 ), (__v8si)(__m256i)(lhs_mat_0123_2), (int)(0))); |
| 1202 | __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_2 ), (__v8si)(__m256i)(lhs_mat_0123_2), (int)(17))); |
| 1203 | __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptrs[rp][b].qs + 96))); |
| 1204 | __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_3 ), (__v8si)(__m256i)(lhs_mat_0123_3), (int)(0))); |
| 1205 | __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_3 ), (__v8si)(__m256i)(lhs_mat_0123_3), (int)(17))); |
| 1206 | |
| 1207 | // Shuffle pattern one - left side input |
| 1208 | const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_0 ), (int)(160))); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) |
| 1209 | const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_0 ), (int)(160))); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) |
| 1210 | |
| 1211 | const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_1 ), (int)(160))); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) |
| 1212 | const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_1 ), (int)(160))); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) |
| 1213 | |
| 1214 | const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_2 ), (int)(160))); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) |
| 1215 | const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_2 ), (int)(160))); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) |
| 1216 | |
| 1217 | const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_3 ), (int)(160))); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) |
| 1218 | const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_3 ), (int)(160))); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) |
| 1219 | |
| 1220 | // Shuffle pattern two - left side input |
| 1221 | const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_0 ), (int)(245))); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) |
| 1222 | const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_0 ), (int)(245))); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) |
| 1223 | |
| 1224 | const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_1 ), (int)(245))); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) |
| 1225 | const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_1 ), (int)(245))); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) |
| 1226 | |
| 1227 | const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_2 ), (int)(245))); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) |
| 1228 | const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_2 ), (int)(245))); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) |
| 1229 | |
| 1230 | const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_3 ), (int)(245))); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) |
| 1231 | const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_3 ), (int)(245))); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) |
| 1232 | |
| 1233 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 1234 | // Resembles MMLAs into 2x2 matrices in ARM Version |
| 1235 | const __m256i zero = _mm256_setzero_si256(); |
| 1236 | __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1); |
| 1237 | __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1); |
| 1238 | __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1); |
| 1239 | __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1); |
| 1240 | __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2); |
| 1241 | __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2); |
| 1242 | __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2); |
| 1243 | __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2); |
| 1244 | |
| 1245 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 1246 | __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2); |
| 1247 | __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2); |
| 1248 | __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2); |
| 1249 | __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2); |
| 1250 | |
| 1251 | // Straighten out to make 4 row vectors |
| 1252 | __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01), (int)(78)))), (int)(204))); |
| 1253 | __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00), (int )(78)))), (__v8si)(__m256i)(iacc_mat_01), (int)(204))); |
| 1254 | __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11), (int)(78)))), (int)(204))); |
| 1255 | __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10), (int )(78)))), (__v8si)(__m256i)(iacc_mat_11), (int)(204))); |
| 1256 | |
| 1257 | // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes |
| 1258 | const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask)_mm256_cvtph_ps(((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i )(_mm_maskload_epi32((int const*)(a_ptrs[rp][b].d), loadMask) ), (int)(68)))); |
| 1259 | |
| 1260 | // Multiply with appropriate scales and accumulate |
| 1261 | acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 1262 | acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 1263 | acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 1264 | acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 1265 | } |
| 1266 | } |
| 1267 | |
| 1268 | // Store the accumulated values |
| 1269 | for (int i = 0; i < 16; i++) { |
| 1270 | _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]); |
| 1271 | } |
| 1272 | } |
| 1273 | } |
| 1274 | |
| 1275 | // Take a block_q8_0x4 structures at each pass of the loop and perform dot product operation |
| 1276 | for (; y < nr / 4; y ++) { |
| 1277 | const block_q8_0x4 * a_ptr = a_ptr_start + (y * nb); |
| 1278 | |
| 1279 | // Load the eight blocks of quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 |
| 1280 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 1281 | const block_tx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 1282 | |
| 1283 | // Master FP accumulators |
| 1284 | __m256 acc_rows[4]; |
| 1285 | for (int i = 0; i < 4; i++) { |
| 1286 | acc_rows[i] = _mm256_setzero_ps(); |
| 1287 | } |
| 1288 | |
| 1289 | for (int64_t b = 0; b < nb; b++) { |
| 1290 | // Load the eight block_q8_0 quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 |
| 1291 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs)); |
| 1292 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32)); |
| 1293 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64)); |
| 1294 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96)); |
| 1295 | |
| 1296 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 1297 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 1298 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 1299 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 1300 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 1301 | |
| 1302 | // 4-bit -> 8-bit - Sign is maintained |
| 1303 | const __m256i rhs_mat_0145_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_0, m4b)); //B0(0-7) B1(0-7) B4(0-7) B5(0-7) |
| 1304 | const __m256i rhs_mat_2367_0 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_0, m4b)); //B2(0-7) B3(0-7) B6(0-7) B7(0-7) |
| 1305 | |
| 1306 | const __m256i rhs_mat_0145_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_0145_1, m4b)); //B0(8-15) B1(8-15) B4(8-15) B5(8-15) |
| 1307 | const __m256i rhs_mat_2367_1 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(rhs_raw_mat_2367_1, m4b)); //B2(8-15) B3(8-15) B6(8-15) B7(8-15) |
| 1308 | |
| 1309 | const __m256i rhs_mat_0145_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b)); //B0(16-23) B1(16-23) B4(16-23) B5(16-23) |
| 1310 | const __m256i rhs_mat_2367_2 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b)); //B2(16-23) B3(16-23) B6(16-23) B7(16-23) |
| 1311 | |
| 1312 | const __m256i rhs_mat_0145_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b)); //B0(24-31) B1(24-31) B4(24-31) B5(24-31) |
| 1313 | const __m256i rhs_mat_2367_3 = _mm256_shuffle_epi8(signextendlut, _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b)); //B2(24-31) B3(24-31) B6(24-31) B7(24-31) |
| 1314 | |
| 1315 | // Shuffle pattern one - right side input |
| 1316 | const __m256i rhs_mat_0145_0_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_0, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_0 ), (int)(136))); //B0(0-3) B1(0-3) B0(0-3) B1(0-3) B4(0-3) B5(0-3) B4(0-3) B5(0-3) |
| 1317 | const __m256i rhs_mat_2367_0_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_0, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_0 ), (int)(136))); //B2(0-3) B3(0-3) B2(0-3) B3(0-3) B6(0-3) B7(0-3) B6(0-3) B7(0-3) |
| 1318 | |
| 1319 | const __m256i rhs_mat_0145_1_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_1, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_1 ), (int)(136))); //B0(8-11) B1(8-11) B0(8-11) B1(8-11) B4(8-11) B5(8-11) B4(8-11) B5(8-11) |
| 1320 | const __m256i rhs_mat_2367_1_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_1, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_1 ), (int)(136))); //B2(8-11) B3(8-11) B2(8-11) B3(8-11) B6(8-11) B7(8-11) B6(8-11) B7(8-11) |
| 1321 | |
| 1322 | const __m256i rhs_mat_0145_2_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_2, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_2 ), (int)(136))); //B0(16-19) B1(16-19) B0(16-19) B1(16-19) B4(16-19) B5(16-19) B4(16-19) B5(16-19) |
| 1323 | const __m256i rhs_mat_2367_2_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_2, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_2 ), (int)(136))); //B2(16-19) B3(16-19) B2(16-19) B3(16-19) B6(16-19) B7(16-19) B6(16-19) B7(16-19) |
| 1324 | |
| 1325 | const __m256i rhs_mat_0145_3_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_3, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_3 ), (int)(136))); //B0(24-27) B1(24-27) B0(24-27) B1(24-27) B4(24-27) B5(24-27) B4(24-27) B5(24-27) |
| 1326 | const __m256i rhs_mat_2367_3_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_3, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_3 ), (int)(136))); //B2(24-27) B3(24-27) B2(24-27) B3(24-27) B6(24-27) B7(24-27) B6(24-27) B7(24-27) |
| 1327 | |
| 1328 | // Shuffle pattern two - right side input |
| 1329 | |
| 1330 | const __m256i rhs_mat_0145_0_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_0, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_0 ), (int)(221))); //B0(4-7) B1(4-7) B0(4-7) B1(4-7) B4(4-7) B5(4-7) B4(4-7) B5(4-7) |
| 1331 | const __m256i rhs_mat_2367_0_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_0, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_0 ), (int)(221))); //B2(4-7) B3(4-7) B2(4-7) B3(4-7) B6(4-7) B7(4-7) B6(4-7) B7(4-7) |
| 1332 | |
| 1333 | const __m256i rhs_mat_0145_1_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_1, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_1 ), (int)(221))); //B0(12-15) B1(12-15) B0(12-15) B1(12-15) B4(12-15) B5(12-15) B4(12-15) B5(12-15) |
| 1334 | const __m256i rhs_mat_2367_1_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_1, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_1 ), (int)(221))); //B2(12-15) B3(12-15) B2(12-15) B3(12-15) B6(12-15) B7(12-15) B6(12-15) B7(12-15) |
| 1335 | |
| 1336 | const __m256i rhs_mat_0145_2_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_2, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_2 ), (int)(221))); //B0(20-23) B1(20-23) B0(20-23) B1(20-23) B4(20-23) B5(20-23) B4(20-23) B5(20-23) |
| 1337 | const __m256i rhs_mat_2367_2_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_2, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_2 ), (int)(221))); //B2(20-23) B3(20-23) B2(20-23) B3(20-23) B6(20-23) B7(20-23) B6(20-23) B7(20-23) |
| 1338 | |
| 1339 | const __m256i rhs_mat_0145_3_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_3, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_3 ), (int)(221))); //B0(28-31) B1(28-31) B0(28-31) B1(28-31) B4(28-31) B5(28-31) B4(28-31) B5(28-31) |
| 1340 | const __m256i rhs_mat_2367_3_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_3, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_3 ), (int)(221))); //B2(28-31) B3(28-31) B2(28-31) B3(28-31) B6(28-31) B7(28-31) B6(28-31) B7(28-31) |
| 1341 | |
| 1342 | // Scale values - Load the wight scale values of block_tx8 |
| 1343 | __m256 col_scale_f32; |
| 1344 | if constexpr ( |
| 1345 | std::is_same_v<block_tx8, block_q4_0x8> || |
| 1346 | std::is_same_v<block_tx8, block_iq4_nlx8>) { |
| 1347 | col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 1348 | } else if constexpr (std::is_same_v<block_tx8, block_mxfp4x8>) { |
| 1349 | col_scale_f32 = _mm256_set_ps( |
| 1350 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[7])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[7])], |
| 1351 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[6])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[6])], |
| 1352 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[5])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[5])], |
| 1353 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[4])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[4])], |
| 1354 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[3])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[3])], |
| 1355 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[2])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[2])], |
| 1356 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[1])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[1])], |
| 1357 | GGML_CPU_E8M0_TO_FP32_HALF(b_ptr[b].e[0])ggml_table_f32_e8m0_half[(uint8_t)(b_ptr[b].e[0])]); |
| 1358 | } |
| 1359 | |
| 1360 | // Load the four blocks of quantized values interleaved with each other in chunks of eight - A0,A1,A2,A3 |
| 1361 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 1362 | __m256i lhs_mat_0123_0 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs))); |
| 1363 | __m256i lhs_mat_01_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_0 ), (__v8si)(__m256i)(lhs_mat_0123_0), (int)(0))); |
| 1364 | __m256i lhs_mat_23_0 = _mm256_permute2f128_si256(lhs_mat_0123_0, lhs_mat_0123_0, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_0 ), (__v8si)(__m256i)(lhs_mat_0123_0), (int)(17))); |
| 1365 | __m256i lhs_mat_0123_1 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 32))); |
| 1366 | __m256i lhs_mat_01_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_1 ), (__v8si)(__m256i)(lhs_mat_0123_1), (int)(0))); |
| 1367 | __m256i lhs_mat_23_1 = _mm256_permute2f128_si256(lhs_mat_0123_1, lhs_mat_0123_1, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_1 ), (__v8si)(__m256i)(lhs_mat_0123_1), (int)(17))); |
| 1368 | __m256i lhs_mat_0123_2 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 64))); |
| 1369 | __m256i lhs_mat_01_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_2 ), (__v8si)(__m256i)(lhs_mat_0123_2), (int)(0))); |
| 1370 | __m256i lhs_mat_23_2 = _mm256_permute2f128_si256(lhs_mat_0123_2, lhs_mat_0123_2, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_2 ), (__v8si)(__m256i)(lhs_mat_0123_2), (int)(17))); |
| 1371 | __m256i lhs_mat_0123_3 = _mm256_loadu_si256((const __m256i *)((a_ptr[b].qs + 96))); |
| 1372 | __m256i lhs_mat_01_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_3 ), (__v8si)(__m256i)(lhs_mat_0123_3), (int)(0))); |
| 1373 | __m256i lhs_mat_23_3 = _mm256_permute2f128_si256(lhs_mat_0123_3, lhs_mat_0123_3, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_3 ), (__v8si)(__m256i)(lhs_mat_0123_3), (int)(17))); |
| 1374 | |
| 1375 | // Shuffle pattern one - left side input |
| 1376 | |
| 1377 | const __m256i lhs_mat_01_0_sp1 = _mm256_shuffle_epi32(lhs_mat_01_0, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_0 ), (int)(160))); //A0(0-3) A0(0-3) A1(0-3) A1(0-3) A0(0-3) A0(0-3) A1(0-3) A1(0-3) |
| 1378 | const __m256i lhs_mat_23_0_sp1 = _mm256_shuffle_epi32(lhs_mat_23_0, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_0 ), (int)(160))); //A2(0-3) A2(0-3) A3(0-3) A3(0-3) A2(0-3) A2(0-3) A3(0-3) A3(0-3) |
| 1379 | |
| 1380 | const __m256i lhs_mat_01_1_sp1 = _mm256_shuffle_epi32(lhs_mat_01_1, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_1 ), (int)(160))); //A0(8-11) A0(8-11) A1(8-11) A1(8-11) A0(8-11) A0(8-11) A1(8-11) A1(8-11) |
| 1381 | const __m256i lhs_mat_23_1_sp1 = _mm256_shuffle_epi32(lhs_mat_23_1, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_1 ), (int)(160))); //A2(8-11) A2(8-11) A3(8-11) A3(8-11) A2(8-11) A2(8-11) A3(8-11) A3(8-11) |
| 1382 | |
| 1383 | const __m256i lhs_mat_01_2_sp1 = _mm256_shuffle_epi32(lhs_mat_01_2, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_2 ), (int)(160))); //A0(16-19) A0(16-19) A1(16-19) A1(16-19) A0(16-19) A0(16-19) A1(16-19) A1(16-19) |
| 1384 | const __m256i lhs_mat_23_2_sp1 = _mm256_shuffle_epi32(lhs_mat_23_2, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_2 ), (int)(160))); //A2(16-19) A2(16-19) A3(16-19) A3(16-19) A2(16-19) A2(16-19) A3(16-19) A3(16-19) |
| 1385 | |
| 1386 | const __m256i lhs_mat_01_3_sp1 = _mm256_shuffle_epi32(lhs_mat_01_3, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_3 ), (int)(160))); //A0(24-27) A0(24-27) A1(24-27) A1(24-27) A0(24-27) A0(24-27) A1(24-27) A1(24-27) |
| 1387 | const __m256i lhs_mat_23_3_sp1 = _mm256_shuffle_epi32(lhs_mat_23_3, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_3 ), (int)(160))); //A2(24-27) A2(24-27) A3(24-27) A3(24-27) A2(24-27) A2(24-27) A3(24-27) A3(24-27) |
| 1388 | |
| 1389 | // Shuffle pattern two - left side input |
| 1390 | |
| 1391 | const __m256i lhs_mat_01_0_sp2 = _mm256_shuffle_epi32(lhs_mat_01_0, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_0 ), (int)(245))); //A0(4-7) A0(4-7) A1(4-7) A1(4-7) A0(4-7) A0(4-7) A1(4-7) A1(4-7) |
| 1392 | const __m256i lhs_mat_23_0_sp2 = _mm256_shuffle_epi32(lhs_mat_23_0, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_0 ), (int)(245))); //A2(4-7) A2(4-7) A3(4-7) A3(4-7) A2(4-7) A2(4-7) A3(4-7) A3(4-7) |
| 1393 | |
| 1394 | const __m256i lhs_mat_01_1_sp2 = _mm256_shuffle_epi32(lhs_mat_01_1, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_1 ), (int)(245))); //A0(12-15) A0(12-15) A1(12-15) A1(12-15) A0(12-15) A0(12-15) A1(12-15) A1(12-15) |
| 1395 | const __m256i lhs_mat_23_1_sp2 = _mm256_shuffle_epi32(lhs_mat_23_1, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_1 ), (int)(245))); //A2(12-15) A2(12-15) A3(12-15) A3(12-15) A2(12-15) A2(12-15) A3(12-15) A3(12-15) |
| 1396 | |
| 1397 | const __m256i lhs_mat_01_2_sp2 = _mm256_shuffle_epi32(lhs_mat_01_2, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_2 ), (int)(245))); //A0(20-23) A0(20-23) A1(20-23) A1(20-23) A0(20-23) A0(20-23) A1(20-23) A1(20-23) |
| 1398 | const __m256i lhs_mat_23_2_sp2 = _mm256_shuffle_epi32(lhs_mat_23_2, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_2 ), (int)(245))); //A2(20-23) A2(20-23) A3(20-23) A3(20-23) A2(20-23) A2(20-23) A3(20-23) A3(20-23) |
| 1399 | |
| 1400 | const __m256i lhs_mat_01_3_sp2 = _mm256_shuffle_epi32(lhs_mat_01_3, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_3 ), (int)(245))); //A0(28-31) A0(28-31) A1(28-31) A1(28-31) A0(28-31) A0(28-31) A1(28-31) A1(28-31) |
| 1401 | const __m256i lhs_mat_23_3_sp2 = _mm256_shuffle_epi32(lhs_mat_23_3, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_3 ), (int)(245))); //A2(28-31) A2(28-31) A3(28-31) A3(28-31) A2(28-31) A2(28-31) A3(28-31) A3(28-31) |
| 1402 | |
| 1403 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 1404 | // Resembles MMLAs into 2x2 matrices in ARM Version |
| 1405 | const __m256i zero = _mm256_setzero_si256(); |
| 1406 | __m256i iacc_mat_00_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_01_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_01_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_01_0_sp1, rhs_mat_0145_0_sp1); |
| 1407 | __m256i iacc_mat_01_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_01_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_01_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_01_0_sp1, rhs_mat_2367_0_sp1); |
| 1408 | __m256i iacc_mat_10_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_0145_3_sp1), lhs_mat_23_2_sp1, rhs_mat_0145_2_sp1), lhs_mat_23_1_sp1, rhs_mat_0145_1_sp1), lhs_mat_23_0_sp1, rhs_mat_0145_0_sp1); |
| 1409 | __m256i iacc_mat_11_sp1 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp1, rhs_mat_2367_3_sp1), lhs_mat_23_2_sp1, rhs_mat_2367_2_sp1), lhs_mat_23_1_sp1, rhs_mat_2367_1_sp1), lhs_mat_23_0_sp1, rhs_mat_2367_0_sp1); |
| 1410 | __m256i iacc_mat_00_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_01_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_01_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_01_0_sp2, rhs_mat_0145_0_sp2); |
| 1411 | __m256i iacc_mat_01_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_01_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_01_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_01_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_01_0_sp2, rhs_mat_2367_0_sp2); |
| 1412 | __m256i iacc_mat_10_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_0145_3_sp2), lhs_mat_23_2_sp2, rhs_mat_0145_2_sp2), lhs_mat_23_1_sp2, rhs_mat_0145_1_sp2), lhs_mat_23_0_sp2, rhs_mat_0145_0_sp2); |
| 1413 | __m256i iacc_mat_11_sp2 = mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(mul_sum_i8_pairs_acc_int32x8(zero, lhs_mat_23_3_sp2, rhs_mat_2367_3_sp2), lhs_mat_23_2_sp2, rhs_mat_2367_2_sp2), lhs_mat_23_1_sp2, rhs_mat_2367_1_sp2), lhs_mat_23_0_sp2, rhs_mat_2367_0_sp2); |
| 1414 | |
| 1415 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 1416 | __m256i iacc_mat_00 = _mm256_add_epi32(iacc_mat_00_sp1, iacc_mat_00_sp2); |
| 1417 | __m256i iacc_mat_01 = _mm256_add_epi32(iacc_mat_01_sp1, iacc_mat_01_sp2); |
| 1418 | __m256i iacc_mat_10 = _mm256_add_epi32(iacc_mat_10_sp1, iacc_mat_10_sp2); |
| 1419 | __m256i iacc_mat_11 = _mm256_add_epi32(iacc_mat_11_sp1, iacc_mat_11_sp2); |
| 1420 | |
| 1421 | |
| 1422 | // Straighten out to make 4 row vectors |
| 1423 | __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01), (int)(78)))), (int)(204))); |
| 1424 | __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00), (int )(78)))), (__v8si)(__m256i)(iacc_mat_01), (int)(204))); |
| 1425 | __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11), (int)(78)))), (int)(204))); |
| 1426 | __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10), (int )(78)))), (__v8si)(__m256i)(iacc_mat_11), (int)(204))); |
| 1427 | |
| 1428 | // Load the scale(d) values for all the 4 Q8_0 blocks and repeat it across lanes |
| 1429 | const __m256 row_scale_f32 = GGML_F32Cx8_REPEAT_LOAD(a_ptr[b].d, loadMask)_mm256_cvtph_ps(((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i )(_mm_maskload_epi32((int const*)(a_ptr[b].d), loadMask)), (int )(68)))); |
| 1430 | |
| 1431 | // Multiply with appropriate scales and accumulate |
| 1432 | acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 1433 | acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 1434 | acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 1435 | acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 1436 | } |
| 1437 | |
| 1438 | // Store the accumulated values |
| 1439 | for (int i = 0; i < 4; i++) { |
| 1440 | _mm256_storeu_ps((float *)(s + ((y * 4 + i) * bs + x * 8)), acc_rows[i]); |
| 1441 | } |
| 1442 | } |
| 1443 | } |
| 1444 | } |
| 1445 | |
| 1446 | #endif // defined(__AVX2__) || defined(__AVX512F__) |
| 1447 | |
| 1448 | void ggml_gemv_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 1449 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 1450 | { |
| 1451 | // Lookup table to convert signed nibbles to signed bytes |
| 1452 | __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); |
| 1453 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 1454 | |
| 1455 | gemv_q4_b32_8x8_q8_0_lut_avx<block_q4_0x8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 1456 | |
| 1457 | return; |
| 1458 | } |
| 1459 | #endif |
| 1460 | |
| 1461 | ggml_gemv_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); |
| 1462 | } |
| 1463 | |
| 1464 | void ggml_gemv_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 1465 | const int qk = QK_K256; |
| 1466 | const int nb = n / qk; |
| 1467 | const int ncols_interleaved = 8; |
| 1468 | const int blocklen = 8; |
| 1469 | static const uint32_t kmask1 = 0x3f3f3f3f; |
| 1470 | static const uint32_t kmask2 = 0x0f0f0f0f; |
| 1471 | static const uint32_t kmask3 = 0x03030303; |
| 1472 | |
| 1473 | assert (n % qk == 0)(static_cast <bool> (n % qk == 0) ? void (0) : __assert_fail ("n % qk == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 1474 | assert (nc % ncols_interleaved == 0)(static_cast <bool> (nc % ncols_interleaved == 0) ? void (0) : __assert_fail ("nc % ncols_interleaved == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 1475 | |
| 1476 | UNUSED(s)(void)(s); |
| 1477 | UNUSED(bs)(void)(bs); |
| 1478 | UNUSED(vx)(void)(vx); |
| 1479 | UNUSED(vy)(void)(vy); |
| 1480 | UNUSED(nr)(void)(nr); |
| 1481 | UNUSED(nc)(void)(nc); |
| 1482 | UNUSED(nb)(void)(nb); |
| 1483 | UNUSED(ncols_interleaved)(void)(ncols_interleaved); |
| 1484 | UNUSED(blocklen)(void)(blocklen); |
| 1485 | |
| 1486 | #if defined(__AVX2__1) |
| 1487 | // Lookup table to convert signed nibbles to signed bytes |
| 1488 | __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); |
| 1489 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 1490 | // Shuffle masks to rearrange delta and scale values to multiply with appropriate scales |
| 1491 | __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0); |
| 1492 | __m128i scalemask = _mm_set_epi8(7, 7, 3, 3, 6, 6, 2, 2, 5, 5, 1, 1, 4, 4, 0, 0); |
| 1493 | // Permute mask used for easier vector processing at later stages |
| 1494 | __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); |
| 1495 | |
| 1496 | // Mask to extract nibbles from bytes |
| 1497 | const __m256i m4b = _mm256_set1_epi8(0x0F); |
| 1498 | |
| 1499 | int64_t b_nb = n / QK_K256; |
| 1500 | |
| 1501 | const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 *)vx; |
| 1502 | const block_q8_K * a_ptr_start = (const block_q8_K *)vy; |
| 1503 | |
| 1504 | // Process Q8_K blocks one by one |
| 1505 | for (int64_t y = 0; y < nr; y++) { |
| 1506 | |
| 1507 | // Pointers to LHS blocks of block_q8_K format |
| 1508 | const block_q8_K * a_ptr = a_ptr_start + (y * nb); |
| 1509 | |
| 1510 | // Take group of eight interleaved block_q4_K structures at each pass of the loop and perform dot product operation |
| 1511 | for (int64_t x = 0; x < nc / 8; x++) { |
| 1512 | |
| 1513 | // Pointers to RHS blocks |
| 1514 | const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 1515 | |
| 1516 | // Master FP accumulators |
| 1517 | __m256 acc_row = _mm256_setzero_ps(); |
| 1518 | __m256 acc_min_rows = _mm256_setzero_ps(); |
| 1519 | |
| 1520 | for (int64_t b = 0; b < nb; b++) { |
| 1521 | |
| 1522 | // Load and convert to FP32 scale from block_q8_K |
| 1523 | const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d)); |
| 1524 | |
| 1525 | // Load the scale values for the 8 blocks interleaved in block_q4_Kx8 |
| 1526 | // col_scale_f32 rearranged so as to multiply with appropriate quants |
| 1527 | const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask)_mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) b_ptr[b].d), deltamask)); |
| 1528 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 1529 | |
| 1530 | __m256i iacc_b = _mm256_setzero_si256(); |
| 1531 | __m256i iacc_min_b = _mm256_setzero_si256(); |
| 1532 | |
| 1533 | const __m256i q8sums = _mm256_loadu_si256((const __m256i * )(a_ptr[b].bsums)); |
| 1534 | __m256i q8s = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(q8sums), _mm256_extracti128_si256(q8sums, 1)((__m128i)__builtin_ia32_extract128i256((__v4di)(__m256i)(q8sums ), (int)(1))))); |
| 1535 | q8s = _mm256_permute2f128_si256(q8s, q8s, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(q8s ), (__v8si)(__m256i)(q8s), (int)(0))); |
| 1536 | |
| 1537 | // Processes two sub blocks from each Q4_K in each iteration |
| 1538 | for (int sb = 0; sb < QK_K256 / 64; sb++) { |
| 1539 | |
| 1540 | // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 |
| 1541 | const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); |
| 1542 | const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); |
| 1543 | const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); |
| 1544 | const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); |
| 1545 | const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); |
| 1546 | const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); |
| 1547 | const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); |
| 1548 | const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); |
| 1549 | |
| 1550 | // 4-bit -> 8-bit |
| 1551 | // Values of the first sub block of eight block_q4_K structures for the sb loop |
| 1552 | const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m4b); |
| 1553 | const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m4b); |
| 1554 | const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m4b); |
| 1555 | const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m4b); |
| 1556 | const __m256i rhs_vec_0123_02 = _mm256_and_si256(rhs_raw_vec_0123_2, m4b); |
| 1557 | const __m256i rhs_vec_4567_02 = _mm256_and_si256(rhs_raw_vec_4567_2, m4b); |
| 1558 | const __m256i rhs_vec_0123_03 = _mm256_and_si256(rhs_raw_vec_0123_3, m4b); |
| 1559 | const __m256i rhs_vec_4567_03 = _mm256_and_si256(rhs_raw_vec_4567_3, m4b); |
| 1560 | |
| 1561 | // Values of the second sub block of eight block_q4_K structures when sb = 1 |
| 1562 | const __m256i rhs_vec_0123_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m4b); |
| 1563 | const __m256i rhs_vec_4567_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m4b); |
| 1564 | const __m256i rhs_vec_0123_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m4b); |
| 1565 | const __m256i rhs_vec_4567_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m4b); |
| 1566 | const __m256i rhs_vec_0123_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m4b); |
| 1567 | const __m256i rhs_vec_4567_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m4b); |
| 1568 | const __m256i rhs_vec_0123_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m4b); |
| 1569 | const __m256i rhs_vec_4567_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m4b); |
| 1570 | |
| 1571 | uint32_t utmp_0[4], utmp_1[4]; |
| 1572 | |
| 1573 | // Scales and Mins of corresponding sub blocks from different Q8_K structures are stored together |
| 1574 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 1575 | memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); |
| 1576 | utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); |
| 1577 | const uint32_t uaux_0 = utmp_0[1] & kmask1; |
| 1578 | utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); |
| 1579 | utmp_0[2] = uaux_0; |
| 1580 | utmp_0[0] &= kmask1; |
| 1581 | |
| 1582 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 1583 | memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); |
| 1584 | utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); |
| 1585 | const uint32_t uaux_1 = utmp_1[1] & kmask1; |
| 1586 | utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); |
| 1587 | utmp_1[2] = uaux_1; |
| 1588 | utmp_1[0] &= kmask1; |
| 1589 | |
| 1590 | // Scales of first sub block in the sb loop |
| 1591 | const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); |
| 1592 | __m128i scales_rearrange_0 = _mm_shuffle_epi8(mins_and_scales_0, scalemask); |
| 1593 | __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0); |
| 1594 | |
| 1595 | // Scales of second sub block in the sb loop |
| 1596 | __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); |
| 1597 | __m128i scales_rearrange_1 = _mm_shuffle_epi8(mins_and_scales_1, scalemask); |
| 1598 | __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1); |
| 1599 | |
| 1600 | // Mins of first and second sub block of Q4_K block are arranged side by side |
| 1601 | __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_0 ), (int)(78))), _mm_shuffle_epi32(mins_and_scales_1, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_1 ), (int)(78))))); |
| 1602 | |
| 1603 | // Load the two sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector |
| 1604 | __m256i lhs_vec_00 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 64))); |
| 1605 | __m256i lhs_vec_01 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 64))); |
| 1606 | __m256i lhs_vec_10 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 64))); |
| 1607 | __m256i lhs_vec_11 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 64))); |
| 1608 | |
| 1609 | lhs_vec_00 = _mm256_permute2f128_si256(lhs_vec_00, lhs_vec_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_00 ), (__v8si)(__m256i)(lhs_vec_00), (int)(0))); |
| 1610 | lhs_vec_01 = _mm256_permute2f128_si256(lhs_vec_01, lhs_vec_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_01 ), (__v8si)(__m256i)(lhs_vec_01), (int)(0))); |
| 1611 | lhs_vec_10 = _mm256_permute2f128_si256(lhs_vec_10, lhs_vec_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_10 ), (__v8si)(__m256i)(lhs_vec_10), (int)(0))); |
| 1612 | lhs_vec_11 = _mm256_permute2f128_si256(lhs_vec_11, lhs_vec_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_11 ), (__v8si)(__m256i)(lhs_vec_11), (int)(0))); |
| 1613 | |
| 1614 | // Dot product done within 32 bit lanes and accumulated in the same vector |
| 1615 | // First done for first sub block and then for second sub block in each sb |
| 1616 | // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3) |
| 1617 | // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7) |
| 1618 | // ........................................................................... |
| 1619 | // B0(28-31) B4(28-31) B1(28-31) B5(28-31) B2(28-31) B6(28-31) B3(28-31) B7(28-31) with A0(28-31) |
| 1620 | |
| 1621 | |
| 1622 | __m256i iacc_0 = _mm256_setzero_si256(); |
| 1623 | __m256i iacc_1 = _mm256_setzero_si256(); |
| 1624 | |
| 1625 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_00), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_00, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_00 ), (int)(0))))); |
| 1626 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_00), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_00), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_00, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_00 ), (int)(85))))); |
| 1627 | |
| 1628 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_01 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_01), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_00, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_00 ), (int)(170))))); |
| 1629 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_01), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_01), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_00, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_00 ), (int)(255))))); |
| 1630 | |
| 1631 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_02 ,_mm256_shuffle_epi32(rhs_vec_4567_02, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_02 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_02), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_01, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_01 ), (int)(0))))); |
| 1632 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_02, 177) ,rhs_vec_4567_02, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_02), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_02), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_01, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_01 ), (int)(85))))); |
| 1633 | |
| 1634 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_03 ,_mm256_shuffle_epi32(rhs_vec_4567_03, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_03 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_03), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_01, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_01 ), (int)(170))))); |
| 1635 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_03, 177) ,rhs_vec_4567_03, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_03), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_03), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_01, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_01 ), (int)(255))))); |
| 1636 | |
| 1637 | iacc_0 = _mm256_madd_epi16(iacc_0, scales_0); |
| 1638 | |
| 1639 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_10), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_10, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_10 ), (int)(0))))); |
| 1640 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_10), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_10), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_10, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_10 ), (int)(85))))); |
| 1641 | |
| 1642 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_11 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_11), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_10, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_10 ), (int)(170))))); |
| 1643 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_11), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_11), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_10, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_10 ), (int)(255))))); |
| 1644 | |
| 1645 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_12 ,_mm256_shuffle_epi32(rhs_vec_4567_12, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_12 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_12), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_11, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_11 ), (int)(0))))); |
| 1646 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_12, 177) ,rhs_vec_4567_12, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_12), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_12), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_11, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_11 ), (int)(85))))); |
| 1647 | |
| 1648 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_13 ,_mm256_shuffle_epi32(rhs_vec_4567_13, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_13 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_13), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_11, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_11 ), (int)(170))))); |
| 1649 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_13, 177) ,rhs_vec_4567_13, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_13), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_13), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_11, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_11 ), (int)(255))))); |
| 1650 | |
| 1651 | iacc_1 = _mm256_madd_epi16(iacc_1, scales_1); |
| 1652 | |
| 1653 | // Accumulate the iacc value for one sb |
| 1654 | __m256i iacc_sb = _mm256_add_epi32(iacc_0, iacc_1); |
| 1655 | |
| 1656 | // Broadcast the bsums of the two sub blocks of the iteration of Q8_K across the vector |
| 1657 | // Multiply-Add with corresponding mins of Q4_Kx8 with bsums |
| 1658 | __m256i q8s_sb = _mm256_shuffle_epi32(q8s, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(q8s), (int )(0))); |
| 1659 | __m256i iacc_min_sb = _mm256_madd_epi16(q8s_sb, mins_01); |
| 1660 | q8s = _mm256_bsrli_epi128(q8s, 4)((__m256i)__builtin_ia32_psrldqi256_byteshift((__v32qi)(__m256i )(q8s), (int)(4))); |
| 1661 | |
| 1662 | // Accumulate for the complete block |
| 1663 | iacc_b = _mm256_add_epi32(iacc_b, iacc_sb); |
| 1664 | iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb); |
| 1665 | } |
| 1666 | |
| 1667 | // Multiply-Add with scale values for the complete super block |
| 1668 | acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row); |
| 1669 | acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows); |
| 1670 | |
| 1671 | } |
| 1672 | |
| 1673 | // Accumulated output values permuted so as to be stored in appropriate order post accumulation |
| 1674 | acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask); |
| 1675 | _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows)); |
| 1676 | } |
| 1677 | } |
| 1678 | |
| 1679 | #else |
| 1680 | UNUSED(kmask1)(void)(kmask1); |
| 1681 | UNUSED(kmask2)(void)(kmask2); |
| 1682 | UNUSED(kmask3)(void)(kmask3); |
| 1683 | ggml_gemv_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc); |
| 1684 | #endif |
| 1685 | } |
| 1686 | |
| 1687 | void ggml_gemv_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 1688 | #if defined(__AVX2__1) |
| 1689 | __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl)); |
| 1690 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 1691 | |
| 1692 | gemv_q4_b32_8x8_q8_0_lut_avx<block_iq4_nlx8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 1693 | |
| 1694 | return; |
| 1695 | #endif |
| 1696 | |
| 1697 | ggml_gemv_iq4_nl_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); |
| 1698 | } |
| 1699 | |
| 1700 | void ggml_gemv_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 1701 | #if defined(__AVX2__1) |
| 1702 | __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4)); |
| 1703 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 1704 | |
| 1705 | gemv_q4_b32_8x8_q8_0_lut_avx<block_mxfp4x8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 1706 | |
| 1707 | return; |
| 1708 | #endif |
| 1709 | |
| 1710 | ggml_gemv_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); |
| 1711 | } |
| 1712 | |
| 1713 | void ggml_gemv_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 1714 | const int qk = QK_K256; |
| 1715 | const int nb = n / qk; |
| 1716 | const int ncols_interleaved = 8; |
| 1717 | const int blocklen = 8; |
| 1718 | |
| 1719 | assert (n % qk == 0)(static_cast <bool> (n % qk == 0) ? void (0) : __assert_fail ("n % qk == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 1720 | assert (nc % ncols_interleaved == 0)(static_cast <bool> (nc % ncols_interleaved == 0) ? void (0) : __assert_fail ("nc % ncols_interleaved == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 1721 | |
| 1722 | UNUSED(s)(void)(s); |
| 1723 | UNUSED(bs)(void)(bs); |
| 1724 | UNUSED(vx)(void)(vx); |
| 1725 | UNUSED(vy)(void)(vy); |
| 1726 | UNUSED(nr)(void)(nr); |
| 1727 | UNUSED(nc)(void)(nc); |
| 1728 | UNUSED(nb)(void)(nb); |
| 1729 | UNUSED(ncols_interleaved)(void)(ncols_interleaved); |
| 1730 | UNUSED(blocklen)(void)(blocklen); |
| 1731 | |
| 1732 | #if defined(__AVX2__1) |
| 1733 | // Lookup table to convert signed nibbles to signed bytes |
| 1734 | __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); |
| 1735 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 1736 | // Shuffle masks to rearrange delta values to multiply with appropriate scales |
| 1737 | __m128i deltamask = _mm_set_epi8(15, 14, 7, 6, 13, 12, 5, 4, 11, 10, 3, 2, 9, 8, 1, 0); |
| 1738 | // Permute mask used for easier vector processing at later stages |
| 1739 | __m256i finalpermutemask = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); |
| 1740 | |
| 1741 | const __m256i m3b = _mm256_set1_epi8(3); |
| 1742 | const __m128i m4b_sse = _mm_set1_epi8(0xF); |
| 1743 | |
| 1744 | //Mask to get appropriate scales |
| 1745 | __m128i scalemask1 = _mm_set_epi8(14,14,6,6,12,12,4,4,10,10,2,2,8,8,0,0); |
| 1746 | __m128i scalemask2 = _mm_set_epi8(15,15,7,7,13,13,5,5,11,11,3,3,9,9,1,1); |
| 1747 | |
| 1748 | int64_t b_nb = n / QK_K256; |
| 1749 | |
| 1750 | const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 *)vx; |
| 1751 | const block_q8_K * a_ptr_start = (const block_q8_K *)vy; |
| 1752 | |
| 1753 | // Process Q8_K blocks one by one |
| 1754 | for (int64_t y = 0; y < nr; y++) { |
| 1755 | |
| 1756 | // Pointers to LHS blocks of block_q8_K format |
| 1757 | const block_q8_K * a_ptr = a_ptr_start + (y * nb); |
| 1758 | |
| 1759 | // Take group of eight interleaved block_q2_K structures at each pass of the loop and perform dot product operation |
| 1760 | for(int64_t x = 0; x < nc / 8; x++) { |
| 1761 | |
| 1762 | // Pointers to RHS blocks |
| 1763 | const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 1764 | |
| 1765 | // Master FP accumulators |
| 1766 | __m256 acc_row = _mm256_setzero_ps(); |
| 1767 | __m256 acc_min_rows = _mm256_setzero_ps(); |
| 1768 | |
| 1769 | for (int64_t b = 0; b < nb; b++) { |
| 1770 | |
| 1771 | // Load and convert to FP32 delta from block_q8_K |
| 1772 | const __m256 row_scale_f32 = _mm256_set1_ps((a_ptr[b].d)); |
| 1773 | |
| 1774 | // Load the delta values for the 8 blocks interleaved in block_q2_Kx8 |
| 1775 | // col_scale_f32 rearranged so as to multiply with appropriate quants |
| 1776 | const __m256 col_scale_f32 = GGML_F32Cx8_REARRANGE_LOAD(b_ptr[b].d, deltamask)_mm256_cvtph_ps(_mm_shuffle_epi8(_mm_loadu_si128((const __m128i *) b_ptr[b].d), deltamask)); |
| 1777 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 1778 | |
| 1779 | __m256i iacc_b = _mm256_setzero_si256(); |
| 1780 | __m256i iacc_min_b = _mm256_setzero_si256(); |
| 1781 | |
| 1782 | // Processes eight sub blocks from each Q2_K in each iteration |
| 1783 | for(int sb = 0; sb < QK_K256 / 128; sb++) { |
| 1784 | |
| 1785 | // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight - B0,B1 ....B6,B7 |
| 1786 | const __m256i rhs_raw_vec_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); |
| 1787 | const __m256i rhs_raw_vec_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); |
| 1788 | const __m256i rhs_raw_vec_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); |
| 1789 | const __m256i rhs_raw_vec_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); |
| 1790 | const __m256i rhs_raw_vec_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); |
| 1791 | const __m256i rhs_raw_vec_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); |
| 1792 | const __m256i rhs_raw_vec_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); |
| 1793 | const __m256i rhs_raw_vec_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); |
| 1794 | |
| 1795 | // 2-bit -> 8-bit |
| 1796 | // Values of the 0th,2nd,4th,6th sub blocks of eight block_q2_K structures for the sb loop |
| 1797 | const __m256i rhs_vec_0123_00 = _mm256_and_si256(rhs_raw_vec_0123_0, m3b); //B00(0-7) B01(0-7) B02(0-7) B03(0-7) |
| 1798 | const __m256i rhs_vec_0123_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 2), m3b); //B20(0-7) B21(0-7) B22(0-7) B23(0-7) |
| 1799 | const __m256i rhs_vec_0123_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 4), m3b); //B40(0-7) B41(0-7) B42(0-7) B43(0-7) |
| 1800 | const __m256i rhs_vec_0123_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_0, 6), m3b); //B60(0-7) B61(0-7) B62(0-7) B63(0-7) |
| 1801 | |
| 1802 | const __m256i rhs_vec_4567_00 = _mm256_and_si256(rhs_raw_vec_4567_0, m3b); //B04(0-7) B05(0-7) B06(0-7) B07(0-7) |
| 1803 | const __m256i rhs_vec_4567_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 2), m3b); //B24(0-7) B25(0-7) B26(0-7) B27(0-7) |
| 1804 | const __m256i rhs_vec_4567_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 4), m3b); //B44(0-7) B45(0-7) B46(0-7) B47(0-7) |
| 1805 | const __m256i rhs_vec_4567_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_0, 6), m3b); //B64(0-7) B65(0-7) B66(0-7) B67(0-7) |
| 1806 | |
| 1807 | const __m256i rhs_vec_0123_01 = _mm256_and_si256(rhs_raw_vec_0123_1, m3b); //B00(8-15) B01(8-15) B02(8-15) B03(8-15) |
| 1808 | const __m256i rhs_vec_0123_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 2), m3b); //B20(8-15) B21(8-15) B22(8-15) B23(8-15) |
| 1809 | const __m256i rhs_vec_0123_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 4), m3b); //B40(8-15) B41(8-15) B42(8-15) B43(8-15) |
| 1810 | const __m256i rhs_vec_0123_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_1, 6), m3b); //B60(8-15) B61(8-15) B62(8-15) B63(8-15) |
| 1811 | |
| 1812 | const __m256i rhs_vec_4567_01 = _mm256_and_si256(rhs_raw_vec_4567_1, m3b); //B04(8-15) B05(8-15) B06(8-15) B07(8-15) |
| 1813 | const __m256i rhs_vec_4567_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 2), m3b); //B24(8-15) B25(8-15) B26(8-15) B27(8-15) |
| 1814 | const __m256i rhs_vec_4567_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 4), m3b); //B44(8-15) B45(8-15) B46(8-15) B47(8-15) |
| 1815 | const __m256i rhs_vec_4567_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_1, 6), m3b); //B64(8-15) B65(8-15) B66(8-15) B67(8-15) |
| 1816 | |
| 1817 | // Values of the 1st,3rd,5th,7th sub blocks of eight block_q2_K structures for the sb loop |
| 1818 | const __m256i rhs_vec_0123_10 = _mm256_and_si256(rhs_raw_vec_0123_2, m3b); //B10(0-7) B11(0-7) B12(0-7) B13(0-7) |
| 1819 | const __m256i rhs_vec_0123_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 2), m3b); //B30(0-7) B31(0-7) B32(0-7) B33(0-7) |
| 1820 | const __m256i rhs_vec_0123_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 4), m3b); //B50(0-7) B51(0-7) B52(0-7) B53(0-7) |
| 1821 | const __m256i rhs_vec_0123_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_2, 6), m3b); //B70(0-7) B71(0-7) B72(0-7) B73(0-7) |
| 1822 | |
| 1823 | const __m256i rhs_vec_4567_10 = _mm256_and_si256(rhs_raw_vec_4567_2, m3b); //B14(0-7) B15(0-7) B16(0-7) B17(0-7) |
| 1824 | const __m256i rhs_vec_4567_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 2), m3b); //B34(0-7) B35(0-7) B36(0-7) B37(0-7) |
| 1825 | const __m256i rhs_vec_4567_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 4), m3b); //B54(0-7) B55(0-7) B56(0-7) B57(0-7) |
| 1826 | const __m256i rhs_vec_4567_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_2, 6), m3b); //B74(0-7) B75(0-7) B76(0-7) B77(0-7) |
| 1827 | |
| 1828 | const __m256i rhs_vec_0123_11 = _mm256_and_si256(rhs_raw_vec_0123_3, m3b); //B10(8-15) B11(8-15) B12(8-15) B13(8-15) |
| 1829 | const __m256i rhs_vec_0123_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 2), m3b); //B30(8-15) B31(8-15) B32(8-15) B33(8-15) |
| 1830 | const __m256i rhs_vec_0123_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 4), m3b); //B50(8-15) B51(8-15) B52(8-15) B53(8-15) |
| 1831 | const __m256i rhs_vec_0123_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_0123_3, 6), m3b); //B70(8-15) B71(8-15) B72(8-15) B73(8-15) |
| 1832 | |
| 1833 | const __m256i rhs_vec_4567_11 = _mm256_and_si256(rhs_raw_vec_4567_3, m3b); //B14(8-15) B15(8-15) B16(8-15) B17(8-15) |
| 1834 | const __m256i rhs_vec_4567_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 2), m3b); //B34(8-15) B35(8-15) B36(8-15) B37(8-15) |
| 1835 | const __m256i rhs_vec_4567_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 4), m3b); //B54(8-15) B55(8-15) B56(8-15) B57(8-15) |
| 1836 | const __m256i rhs_vec_4567_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_vec_4567_3, 6), m3b); //B74(8-15) B75(8-15) B76(8-15) B77(8-15) |
| 1837 | |
| 1838 | //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together |
| 1839 | //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71 |
| 1840 | |
| 1841 | const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64)); |
| 1842 | const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64)); |
| 1843 | const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64)); |
| 1844 | const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64)); |
| 1845 | |
| 1846 | // Extract scales which is lower half from mins_and_scales |
| 1847 | const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse); |
| 1848 | const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse); |
| 1849 | const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse); |
| 1850 | const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse); |
| 1851 | |
| 1852 | // Extract mins which is upper half from mins_and_scales |
| 1853 | const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse)); |
| 1854 | const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse)); |
| 1855 | const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse)); |
| 1856 | const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse)); |
| 1857 | |
| 1858 | // Scales of sub blocks in the sb loop |
| 1859 | // Scales of the 0th sub block from each super block |
| 1860 | __m128i scales_rearrange_0 = _mm_shuffle_epi8(scales_01, scalemask1); |
| 1861 | __m256i scales_0 = _mm256_cvtepu8_epi16(scales_rearrange_0); |
| 1862 | |
| 1863 | // Scales of the 1st sub block from each super block |
| 1864 | __m128i scales_rearrange_1 = _mm_shuffle_epi8(scales_01, scalemask2); |
| 1865 | __m256i scales_1 = _mm256_cvtepu8_epi16(scales_rearrange_1); |
| 1866 | |
| 1867 | // Scales of the 2nd sub block from each super block |
| 1868 | __m128i scales_rearrange_2 = _mm_shuffle_epi8(scales_23, scalemask1); |
| 1869 | __m256i scales_2 = _mm256_cvtepu8_epi16(scales_rearrange_2); |
| 1870 | |
| 1871 | // Scales of the 3rd sub block from each super block |
| 1872 | __m128i scales_rearrange_3 = _mm_shuffle_epi8(scales_23, scalemask2); |
| 1873 | __m256i scales_3 = _mm256_cvtepu8_epi16(scales_rearrange_3); |
| 1874 | |
| 1875 | // Scales of the 4th sub block from each super block |
| 1876 | __m128i scales_rearrange_4 = _mm_shuffle_epi8(scales_45, scalemask1); |
| 1877 | __m256i scales_4 = _mm256_cvtepu8_epi16(scales_rearrange_4); |
| 1878 | |
| 1879 | // Scales of the 5th sub block from each super block |
| 1880 | __m128i scales_rearrange_5 = _mm_shuffle_epi8(scales_45, scalemask2); |
| 1881 | __m256i scales_5 = _mm256_cvtepu8_epi16(scales_rearrange_5); |
| 1882 | |
| 1883 | // Scales of the 6th sub block from each super block |
| 1884 | __m128i scales_rearrange_6 = _mm_shuffle_epi8(scales_67, scalemask1); |
| 1885 | __m256i scales_6 = _mm256_cvtepu8_epi16(scales_rearrange_6); |
| 1886 | |
| 1887 | // Scales of the 7th sub block from each super block |
| 1888 | __m128i scales_rearrange_7 = _mm_shuffle_epi8(scales_67, scalemask2); |
| 1889 | __m256i scales_7 = _mm256_cvtepu8_epi16(scales_rearrange_7); |
| 1890 | |
| 1891 | // Load the sub block values corresponding to sb in block_q8_K in batches of 16 bytes and replicate the same across 256 bit vector |
| 1892 | __m256i lhs_vec_0 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + sb * 128))); |
| 1893 | __m256i lhs_vec_1 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 16 + sb * 128))); |
| 1894 | __m256i lhs_vec_2 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 32 + sb * 128))); |
| 1895 | __m256i lhs_vec_3 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 48 + sb * 128))); |
| 1896 | __m256i lhs_vec_4 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 64 + sb * 128))); |
| 1897 | __m256i lhs_vec_5 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 80 + sb * 128))); |
| 1898 | __m256i lhs_vec_6 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 96 + sb * 128))); |
| 1899 | __m256i lhs_vec_7 = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i *)(a_ptr[b].qs + 112 + sb * 128))); |
| 1900 | |
| 1901 | lhs_vec_0 = _mm256_permute2f128_si256(lhs_vec_0, lhs_vec_0, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_0 ), (__v8si)(__m256i)(lhs_vec_0), (int)(0))); |
| 1902 | lhs_vec_1 = _mm256_permute2f128_si256(lhs_vec_1, lhs_vec_1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_1 ), (__v8si)(__m256i)(lhs_vec_1), (int)(0))); |
| 1903 | lhs_vec_2 = _mm256_permute2f128_si256(lhs_vec_2, lhs_vec_2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_2 ), (__v8si)(__m256i)(lhs_vec_2), (int)(0))); |
| 1904 | lhs_vec_3 = _mm256_permute2f128_si256(lhs_vec_3, lhs_vec_3, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_3 ), (__v8si)(__m256i)(lhs_vec_3), (int)(0))); |
| 1905 | lhs_vec_4 = _mm256_permute2f128_si256(lhs_vec_4, lhs_vec_4, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_4 ), (__v8si)(__m256i)(lhs_vec_4), (int)(0))); |
| 1906 | lhs_vec_5 = _mm256_permute2f128_si256(lhs_vec_5, lhs_vec_5, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_5 ), (__v8si)(__m256i)(lhs_vec_5), (int)(0))); |
| 1907 | lhs_vec_6 = _mm256_permute2f128_si256(lhs_vec_6, lhs_vec_6, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_6 ), (__v8si)(__m256i)(lhs_vec_6), (int)(0))); |
| 1908 | lhs_vec_7 = _mm256_permute2f128_si256(lhs_vec_7, lhs_vec_7, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_vec_7 ), (__v8si)(__m256i)(lhs_vec_7), (int)(0))); |
| 1909 | |
| 1910 | __m256i iacc_0 = _mm256_setzero_si256(); |
| 1911 | __m256i iacc_1 = _mm256_setzero_si256(); |
| 1912 | __m256i iacc_2 = _mm256_setzero_si256(); |
| 1913 | __m256i iacc_3 = _mm256_setzero_si256(); |
| 1914 | __m256i iacc_4 = _mm256_setzero_si256(); |
| 1915 | __m256i iacc_5 = _mm256_setzero_si256(); |
| 1916 | __m256i iacc_6 = _mm256_setzero_si256(); |
| 1917 | __m256i iacc_7 = _mm256_setzero_si256(); |
| 1918 | |
| 1919 | // Dot product done within 32 bit lanes and accumulated in the same vector |
| 1920 | // First done for 0th sub block and then for seven (1st - 7th) other sub blocks processed for each sb (sb < QK_K/128 loop) // B0(0-3) B4(0-3) B1(0-3) B5(0-3) B2(0-3) B6(0-3) B3(0-3) B7(0-3) with A0(0-3) |
| 1921 | // B0(4-7) B4(4-7) B1(4-7) B5(4-7) B2(4-7) B6(4-7) B3(4-7) B7(4-7) with A0(4-7) |
| 1922 | // B0(8-11) B4(8-11) B1(8-11) B5(8-11) B2(8-11) B6(8-11) B3(8-11) B7(8-11) with A0(8-11) |
| 1923 | // B0(12-15) B4(12-15) B1(12-15) B5(12-15) B2(12-15) B6(12-15) B3(12-15) B7(12-15) with A0(12-15) |
| 1924 | |
| 1925 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_00 ,_mm256_shuffle_epi32(rhs_vec_4567_00, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_00), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(0))))); |
| 1926 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_00, 177) ,rhs_vec_4567_00, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_00), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_00), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_0, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(85))))); |
| 1927 | |
| 1928 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_01 ,_mm256_shuffle_epi32(rhs_vec_4567_01, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_01 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_01), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_0, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(170))))); |
| 1929 | iacc_0 = _mm256_add_epi16(iacc_0, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_01, 177) ,rhs_vec_4567_01, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_01), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_01), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_0, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_0 ), (int)(255))))); |
| 1930 | |
| 1931 | iacc_0 = _mm256_madd_epi16(iacc_0, scales_0); |
| 1932 | |
| 1933 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_10 ,_mm256_shuffle_epi32(rhs_vec_4567_10, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_10), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(0))))); |
| 1934 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_10, 177) ,rhs_vec_4567_10, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_10), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_10), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_1, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(85))))); |
| 1935 | |
| 1936 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_11 ,_mm256_shuffle_epi32(rhs_vec_4567_11, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_11 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_11), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_1, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(170))))); |
| 1937 | iacc_1 = _mm256_add_epi16(iacc_1, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_11, 177) ,rhs_vec_4567_11, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_11), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_11), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_1, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_1 ), (int)(255))))); |
| 1938 | |
| 1939 | iacc_1 = _mm256_madd_epi16(iacc_1, scales_1); |
| 1940 | |
| 1941 | iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_20 ,_mm256_shuffle_epi32(rhs_vec_4567_20, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_20 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_20), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_2, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_2 ), (int)(0))))); |
| 1942 | iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_20, 177) ,rhs_vec_4567_20, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_20), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_20), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_2, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_2 ), (int)(85))))); |
| 1943 | |
| 1944 | iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_21 ,_mm256_shuffle_epi32(rhs_vec_4567_21, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_21 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_21), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_2, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_2 ), (int)(170))))); |
| 1945 | iacc_2 = _mm256_add_epi16(iacc_2, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_21, 177) ,rhs_vec_4567_21, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_21), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_21), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_2, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_2 ), (int)(255))))); |
| 1946 | |
| 1947 | iacc_2 = _mm256_madd_epi16(iacc_2, scales_2); |
| 1948 | |
| 1949 | iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_30 ,_mm256_shuffle_epi32(rhs_vec_4567_30, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_30 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_30), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_3, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_3 ), (int)(0))))); |
| 1950 | iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_30, 177) ,rhs_vec_4567_30, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_30), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_30), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_3, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_3 ), (int)(85))))); |
| 1951 | |
| 1952 | iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_31 ,_mm256_shuffle_epi32(rhs_vec_4567_31, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_31 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_31), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_3, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_3 ), (int)(170))))); |
| 1953 | iacc_3 = _mm256_add_epi16(iacc_3, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_31, 177) ,rhs_vec_4567_31, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_31), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_31), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_3, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_3 ), (int)(255))))); |
| 1954 | |
| 1955 | iacc_3 = _mm256_madd_epi16(iacc_3, scales_3); |
| 1956 | |
| 1957 | iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_40 ,_mm256_shuffle_epi32(rhs_vec_4567_40, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_40 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_40), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_4, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_4 ), (int)(0))))); |
| 1958 | iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_40, 177) ,rhs_vec_4567_40, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_40), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_40), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_4, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_4 ), (int)(85))))); |
| 1959 | |
| 1960 | iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_41 ,_mm256_shuffle_epi32(rhs_vec_4567_41, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_41 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_41), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_4, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_4 ), (int)(170))))); |
| 1961 | iacc_4 = _mm256_add_epi16(iacc_4, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_41, 177) ,rhs_vec_4567_41, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_41), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_41), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_4, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_4 ), (int)(255))))); |
| 1962 | |
| 1963 | iacc_4 = _mm256_madd_epi16(iacc_4, scales_4); |
| 1964 | |
| 1965 | iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_50 ,_mm256_shuffle_epi32(rhs_vec_4567_50, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_50 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_50), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_5, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_5 ), (int)(0))))); |
| 1966 | iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_50, 177) ,rhs_vec_4567_50, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_50), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_50), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_5, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_5 ), (int)(85))))); |
| 1967 | |
| 1968 | iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_51 ,_mm256_shuffle_epi32(rhs_vec_4567_51, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_51 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_51), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_5, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_5 ), (int)(170))))); |
| 1969 | iacc_5 = _mm256_add_epi16(iacc_5, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_51, 177) ,rhs_vec_4567_51, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_51), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_51), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_5, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_5 ), (int)(255))))); |
| 1970 | |
| 1971 | iacc_5 = _mm256_madd_epi16(iacc_5, scales_5); |
| 1972 | |
| 1973 | iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_60 ,_mm256_shuffle_epi32(rhs_vec_4567_60, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_60 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_60), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_6, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_6 ), (int)(0))))); |
| 1974 | iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_60, 177) ,rhs_vec_4567_60, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_60), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_60), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_6, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_6 ), (int)(85))))); |
| 1975 | |
| 1976 | iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_61 ,_mm256_shuffle_epi32(rhs_vec_4567_61, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_61 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_61), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_6, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_6 ), (int)(170))))); |
| 1977 | iacc_6 = _mm256_add_epi16(iacc_6, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_61, 177) ,rhs_vec_4567_61, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_61), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_61), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_6, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_6 ), (int)(255))))); |
| 1978 | |
| 1979 | iacc_6 = _mm256_madd_epi16(iacc_6, scales_6); |
| 1980 | |
| 1981 | iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_70 ,_mm256_shuffle_epi32(rhs_vec_4567_70, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_70 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_70), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_7, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_7 ), (int)(0))))); |
| 1982 | iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_70, 177) ,rhs_vec_4567_70, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_70), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_70), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_7, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_7 ), (int)(85))))); |
| 1983 | |
| 1984 | iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(rhs_vec_0123_71 ,_mm256_shuffle_epi32(rhs_vec_4567_71, 177), 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_vec_0123_71 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(rhs_vec_4567_71), (int)(177)))), (int)(170))), _mm256_shuffle_epi32(lhs_vec_7, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_7 ), (int)(170))))); |
| 1985 | iacc_7 = _mm256_add_epi16(iacc_7, _mm256_maddubs_epi16(_mm256_blend_epi32(_mm256_shuffle_epi32(rhs_vec_0123_71, 177) ,rhs_vec_4567_71, 170)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_vec_0123_71), (int)(177)))), (__v8si)(__m256i)(rhs_vec_4567_71), (int)(170 ))), _mm256_shuffle_epi32(lhs_vec_7, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_vec_7 ), (int)(255))))); |
| 1986 | |
| 1987 | iacc_7 = _mm256_madd_epi16(iacc_7, scales_7); |
| 1988 | |
| 1989 | // Accumulate the iacc value for one sb |
| 1990 | __m256i iacc_sb = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_0, iacc_1), _mm256_add_epi32(iacc_2, iacc_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_4, iacc_5), _mm256_add_epi32(iacc_6, iacc_7))); |
| 1991 | |
| 1992 | __m128i q8sums = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + sb * 8)); |
| 1993 | __m256i q8s = _mm256_castsi128_si256(q8sums); |
| 1994 | q8s= _mm256_permute2f128_si256(q8s, q8s, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(q8s ), (__v8si)(__m256i)(q8s), (int)(0))); |
| 1995 | |
| 1996 | // Broadcast the bsums of the two corresponding subblocks of q8_k |
| 1997 | // Multiply-Add with corresponding mins of Q2_Kx8 with bsums |
| 1998 | __m256i iacc_min_sb_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(q8s), (int )(0))), mins_01); |
| 1999 | __m256i iacc_min_sb_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(q8s), (int )(85))), mins_23); |
| 2000 | __m256i iacc_min_sb_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(q8s), (int )(170))), mins_45); |
| 2001 | __m256i iacc_min_sb_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(q8s, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(q8s), (int )(255))), mins_67); |
| 2002 | |
| 2003 | __m256i iacc_min_sb = _mm256_add_epi32(_mm256_add_epi32(iacc_min_sb_01, iacc_min_sb_23), _mm256_add_epi32(iacc_min_sb_45,iacc_min_sb_67)); |
| 2004 | |
| 2005 | // Accumulate for the complete block |
| 2006 | iacc_b = _mm256_add_epi32(iacc_b, iacc_sb); |
| 2007 | iacc_min_b = _mm256_add_epi32(iacc_min_b, iacc_min_sb); |
| 2008 | } |
| 2009 | |
| 2010 | //Multiply-Add with scale values for complete super block |
| 2011 | acc_row = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_b), _mm256_mul_ps(col_scale_f32, row_scale_f32), acc_row); |
| 2012 | acc_min_rows = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_min_b), _mm256_mul_ps(col_dmin_f32, row_scale_f32), acc_min_rows); |
| 2013 | } |
| 2014 | // Accumulated output values permuted so as to be stored in appropriate order post accumulation |
| 2015 | acc_row = _mm256_permutevar8x32_ps(acc_row, finalpermutemask); |
| 2016 | _mm256_storeu_ps(s + (y * nr + x * 8), _mm256_sub_ps(acc_row, acc_min_rows)); |
| 2017 | } |
| 2018 | } |
| 2019 | #else |
| 2020 | |
| 2021 | ggml_gemv_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc); |
| 2022 | |
| 2023 | #endif |
| 2024 | } |
| 2025 | |
| 2026 | void ggml_gemm_q4_0_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 2027 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 2028 | { |
| 2029 | // Lookup table to convert signed nibbles to signed bytes |
| 2030 | __m256i signextendlut = _mm256_castsi128_si256(_mm_set_epi8(-1, -2, -3, -4, -5, -6, -7, -8, 7, 6, 5, 4, 3, 2, 1, 0)); |
| 2031 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 2032 | |
| 2033 | gemm_q4_b32_8x8_q8_0_lut_avx<block_q4_0x8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 2034 | |
| 2035 | return; |
| 2036 | } |
| 2037 | #endif // defined(__AVX2__) || defined(__AVX512F__) |
| 2038 | |
| 2039 | ggml_gemm_q4_0_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); |
| 2040 | } |
| 2041 | |
| 2042 | void ggml_gemm_q4_K_8x8_q8_K(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 2043 | const int qk = QK_K256; |
| 2044 | const int nb = n / qk; |
| 2045 | const int ncols_interleaved = 8; |
| 2046 | const int blocklen = 8; |
| 2047 | static const uint32_t kmask1 = 0x3f3f3f3f; |
| 2048 | static const uint32_t kmask2 = 0x0f0f0f0f; |
| 2049 | static const uint32_t kmask3 = 0x03030303; |
| 2050 | |
| 2051 | assert (n % qk == 0)(static_cast <bool> (n % qk == 0) ? void (0) : __assert_fail ("n % qk == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2052 | assert (nr % 4 == 0)(static_cast <bool> (nr % 4 == 0) ? void (0) : __assert_fail ("nr % 4 == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2053 | assert (nc % ncols_interleaved == 0)(static_cast <bool> (nc % ncols_interleaved == 0) ? void (0) : __assert_fail ("nc % ncols_interleaved == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2054 | |
| 2055 | UNUSED(s)(void)(s); |
| 2056 | UNUSED(bs)(void)(bs); |
| 2057 | UNUSED(vx)(void)(vx); |
| 2058 | UNUSED(vy)(void)(vy); |
| 2059 | UNUSED(nr)(void)(nr); |
| 2060 | UNUSED(nc)(void)(nc); |
| 2061 | UNUSED(nb)(void)(nb); |
| 2062 | UNUSED(ncols_interleaved)(void)(ncols_interleaved); |
| 2063 | UNUSED(blocklen)(void)(blocklen); |
| 2064 | |
| 2065 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 2066 | const block_q4_Kx8 * b_ptr_start = (const block_q4_Kx8 * ) vx; |
| 2067 | const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy; |
| 2068 | int64_t b_nb = n / QK_K256; |
| 2069 | int64_t y = 0; |
| 2070 | |
| 2071 | // Mask to mask out nibbles from packed bytes |
| 2072 | const __m256i m4b = _mm256_set1_epi8(0x0F); |
| 2073 | // Permute mask used for easier vector processing at later stages |
| 2074 | __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4); |
| 2075 | int64_t xstart = 0; |
| 2076 | int anr = nr - nr % 16;; // Used to align nr with boundary of 16 |
| 2077 | #if defined(__AVX512BW__) && defined(__AVX512DQ__) |
| 2078 | int anc = nc - nc % 16; // Used to align nc with boundary of 16 |
| 2079 | // Mask to mask out nibbles from packed bytes expanded to 512 bit length |
| 2080 | const __m512i m4bexpanded = _mm512_set1_epi8(0x0F); |
| 2081 | //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation |
| 2082 | for (; y < anr / 4; y += 4) { |
| 2083 | |
| 2084 | const block_q8_Kx4 * a_ptrs[4]; |
| 2085 | |
| 2086 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 2087 | for (int i = 0; i < 3; ++i) { |
| 2088 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 2089 | } |
| 2090 | |
| 2091 | // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation |
| 2092 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 2093 | |
| 2094 | const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 2095 | const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 2096 | |
| 2097 | // Master FP accumulators |
| 2098 | __m512 acc_rows[16]; |
| 2099 | for (int i = 0; i < 16; i++) { |
| 2100 | acc_rows[i] = _mm512_setzero_ps(); |
| 2101 | } |
| 2102 | |
| 2103 | __m512 acc_min_rows[16]; |
| 2104 | for (int i = 0; i < 16; i++) { |
| 2105 | acc_min_rows[i] = _mm512_setzero_ps(); |
| 2106 | } |
| 2107 | |
| 2108 | // For super block |
| 2109 | for (int64_t b = 0; b < nb; b++) { |
| 2110 | // Scale values - Load the sixteen scale values from two block_q4_kx8 structures |
| 2111 | const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 2112 | |
| 2113 | // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures |
| 2114 | const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin); |
| 2115 | |
| 2116 | // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration |
| 2117 | for (int sb = 0; sb < QK_K256 / 64; sb++) { |
| 2118 | |
| 2119 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256)); |
| 2120 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256)); |
| 2121 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256)); |
| 2122 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256)); |
| 2123 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256)); |
| 2124 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256)); |
| 2125 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256)); |
| 2126 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256)); |
| 2127 | |
| 2128 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256)); |
| 2129 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256)); |
| 2130 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256)); |
| 2131 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256)); |
| 2132 | const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256)); |
| 2133 | const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256)); |
| 2134 | const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256)); |
| 2135 | const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256)); |
| 2136 | |
| 2137 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 2138 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 2139 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 2140 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 2141 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 2142 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 2143 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 2144 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 2145 | |
| 2146 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 2147 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 2148 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 2149 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 2150 | const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2 , requiredOrder)), (int)(240))); |
| 2151 | const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_2 ), (int)(240))); |
| 2152 | const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3 , requiredOrder)), (int)(240))); |
| 2153 | const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_3 ), (int)(240))); |
| 2154 | |
| 2155 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 2156 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 2157 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 2158 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 2159 | |
| 2160 | const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_2)), (__v8si)(__m256i)(rhs_raw_mat_89CD_2), (int)(1))); |
| 2161 | const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_2)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_2), (int)(1))); |
| 2162 | const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_3)), (__v8si)(__m256i)(rhs_raw_mat_89CD_3), (int)(1))); |
| 2163 | const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_3)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_3), (int)(1))); |
| 2164 | |
| 2165 | //4-bit -> 8-bit |
| 2166 | const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7) |
| 2167 | const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7) |
| 2168 | const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15) |
| 2169 | const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15) |
| 2170 | |
| 2171 | const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23) |
| 2172 | const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23) |
| 2173 | const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31) |
| 2174 | const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31) |
| 2175 | |
| 2176 | const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7) |
| 2177 | const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7) |
| 2178 | const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15) |
| 2179 | const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15) |
| 2180 | |
| 2181 | const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23) |
| 2182 | const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23) |
| 2183 | const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31) |
| 2184 | const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31) |
| 2185 | |
| 2186 | // Shuffle pattern one - right side input |
| 2187 | const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3) |
| 2188 | const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3) |
| 2189 | const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 2190 | const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11) |
| 2191 | const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_02 ), (int)((_MM_PERM_ENUM)136))); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19) |
| 2192 | const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_02 ), (int)((_MM_PERM_ENUM)136))); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19) |
| 2193 | const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_03 ), (int)((_MM_PERM_ENUM)136))); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27) |
| 2194 | const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_03 ), (int)((_MM_PERM_ENUM)136))); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27) |
| 2195 | |
| 2196 | const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3) |
| 2197 | const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3) |
| 2198 | const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11) |
| 2199 | const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11) |
| 2200 | const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_12 ), (int)((_MM_PERM_ENUM)136))); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19) |
| 2201 | const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_12 ), (int)((_MM_PERM_ENUM)136))); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19) |
| 2202 | const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_13 ), (int)((_MM_PERM_ENUM)136))); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27) |
| 2203 | const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_13 ), (int)((_MM_PERM_ENUM)136))); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27) |
| 2204 | |
| 2205 | // Shuffle pattern two - right side input |
| 2206 | const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7) |
| 2207 | const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7) |
| 2208 | const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15) |
| 2209 | const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15) |
| 2210 | const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_02 ), (int)((_MM_PERM_ENUM)221))); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23) |
| 2211 | const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_02 ), (int)((_MM_PERM_ENUM)221))); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23) |
| 2212 | const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_03 ), (int)((_MM_PERM_ENUM)221))); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31) |
| 2213 | const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_03 ), (int)((_MM_PERM_ENUM)221))); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31) |
| 2214 | |
| 2215 | const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7) |
| 2216 | const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7) |
| 2217 | const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15) |
| 2218 | const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15) |
| 2219 | const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_12 ), (int)((_MM_PERM_ENUM)221))); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23) |
| 2220 | const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_12 ), (int)((_MM_PERM_ENUM)221))); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23) |
| 2221 | const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_13 ), (int)((_MM_PERM_ENUM)221))); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31) |
| 2222 | const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_13 ), (int)((_MM_PERM_ENUM)221))); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31) |
| 2223 | |
| 2224 | uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4]; |
| 2225 | |
| 2226 | // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together |
| 2227 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 2228 | memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12); |
| 2229 | utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4); |
| 2230 | const uint32_t uaux_00 = utmp_00[1] & kmask1; |
| 2231 | utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4); |
| 2232 | utmp_00[2] = uaux_00; |
| 2233 | utmp_00[0] &= kmask1; |
| 2234 | |
| 2235 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 2236 | memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12); |
| 2237 | utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4); |
| 2238 | const uint32_t uaux_01 = utmp_01[1] & kmask1; |
| 2239 | utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4); |
| 2240 | utmp_01[2] = uaux_01; |
| 2241 | utmp_01[0] &= kmask1; |
| 2242 | |
| 2243 | memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12); |
| 2244 | utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4); |
| 2245 | const uint32_t uaux_10 = utmp_10[1] & kmask1; |
| 2246 | utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4); |
| 2247 | utmp_10[2] = uaux_10; |
| 2248 | utmp_10[0] &= kmask1; |
| 2249 | |
| 2250 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 2251 | memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12); |
| 2252 | utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4); |
| 2253 | const uint32_t uaux_11 = utmp_11[1] & kmask1; |
| 2254 | utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4); |
| 2255 | utmp_11[2] = uaux_11; |
| 2256 | utmp_11[0] &= kmask1; |
| 2257 | |
| 2258 | // Scales of first sub block in the sb loop |
| 2259 | const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]); |
| 2260 | const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); |
| 2261 | |
| 2262 | // Scales of second sub block in the sb loop |
| 2263 | const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]); |
| 2264 | const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); |
| 2265 | |
| 2266 | // Mins of first and second sub block of Q4_K block are arranged side by side |
| 2267 | const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(mins_and_scales_0 ), (int)(78))), _mm256_shuffle_epi32(mins_and_scales_1, 78)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(mins_and_scales_1 ), (int)(78))))); |
| 2268 | |
| 2269 | const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)68))); |
| 2270 | const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)238))); |
| 2271 | |
| 2272 | const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)68))); |
| 2273 | const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)238))); |
| 2274 | |
| 2275 | for (int rp = 0; rp < 4; rp++) { |
| 2276 | |
| 2277 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 2278 | // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector |
| 2279 | __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb))); |
| 2280 | __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(0))); |
| 2281 | __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(17))); |
| 2282 | __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb))); |
| 2283 | __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(0))); |
| 2284 | __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(17))); |
| 2285 | __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb))); |
| 2286 | __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_02 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_02), (int)(0))); |
| 2287 | __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_02 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_02), (int)(17))); |
| 2288 | __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb))); |
| 2289 | __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_03 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_03), (int)(0))); |
| 2290 | __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_03 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_03), (int)(17))); |
| 2291 | __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb))); |
| 2292 | __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(0))); |
| 2293 | __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(17))); |
| 2294 | __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb))); |
| 2295 | __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(0))); |
| 2296 | __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(17))); |
| 2297 | __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb))); |
| 2298 | __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_12 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_12), (int)(0))); |
| 2299 | __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_12 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_12), (int)(17))); |
| 2300 | __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb))); |
| 2301 | __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_13 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_13), (int)(0))); |
| 2302 | __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_13 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_13), (int)(17))); |
| 2303 | |
| 2304 | __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_00)), (__v8si)(__m256i)(lhs_mat_ymm_01_00), ( int)(1))); |
| 2305 | __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_00)), (__v8si)(__m256i)(lhs_mat_ymm_23_00), ( int)(1))); |
| 2306 | __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_01)), (__v8si)(__m256i)(lhs_mat_ymm_01_01), ( int)(1))); |
| 2307 | __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_01)), (__v8si)(__m256i)(lhs_mat_ymm_23_01), ( int)(1))); |
| 2308 | __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_02)), (__v8si)(__m256i)(lhs_mat_ymm_01_02), ( int)(1))); |
| 2309 | __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_02)), (__v8si)(__m256i)(lhs_mat_ymm_23_02), ( int)(1))); |
| 2310 | __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_03)), (__v8si)(__m256i)(lhs_mat_ymm_01_03), ( int)(1))); |
| 2311 | __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_03)), (__v8si)(__m256i)(lhs_mat_ymm_23_03), ( int)(1))); |
| 2312 | |
| 2313 | __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_10)), (__v8si)(__m256i)(lhs_mat_ymm_01_10), ( int)(1))); |
| 2314 | __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_10)), (__v8si)(__m256i)(lhs_mat_ymm_23_10), ( int)(1))); |
| 2315 | __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_11)), (__v8si)(__m256i)(lhs_mat_ymm_01_11), ( int)(1))); |
| 2316 | __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_11)), (__v8si)(__m256i)(lhs_mat_ymm_23_11), ( int)(1))); |
| 2317 | __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_12)), (__v8si)(__m256i)(lhs_mat_ymm_01_12), ( int)(1))); |
| 2318 | __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_12)), (__v8si)(__m256i)(lhs_mat_ymm_23_12), ( int)(1))); |
| 2319 | __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_13)), (__v8si)(__m256i)(lhs_mat_ymm_01_13), ( int)(1))); |
| 2320 | __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_13)), (__v8si)(__m256i)(lhs_mat_ymm_23_13), ( int)(1))); |
| 2321 | |
| 2322 | // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks |
| 2323 | __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb))); |
| 2324 | __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (lhs_bsums_0123_01), (int)(1))))); |
| 2325 | lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0)((__m256i)__builtin_ia32_permti256((__m256i)(lhs_bsums_hsum_ymm_0123_01 ), (__m256i)(lhs_bsums_hsum_ymm_0123_01), (int)(0))); |
| 2326 | __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_hsum_ymm_0123_01)), (__v8si)(__m256i)(lhs_bsums_hsum_ymm_0123_01 ), (int)(1))); |
| 2327 | |
| 2328 | // Shuffle pattern one - left side input |
| 2329 | const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 2330 | const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)160))); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) |
| 2331 | const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 2332 | const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)160))); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) |
| 2333 | const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_02 ), (int)((_MM_PERM_ENUM)160))); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) |
| 2334 | const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_02 ), (int)((_MM_PERM_ENUM)160))); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) |
| 2335 | const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_03 ), (int)((_MM_PERM_ENUM)160))); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) |
| 2336 | const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_03 ), (int)((_MM_PERM_ENUM)160))); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) |
| 2337 | |
| 2338 | const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 2339 | const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)160))); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) |
| 2340 | const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 2341 | const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)160))); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) |
| 2342 | const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_12 ), (int)((_MM_PERM_ENUM)160))); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) |
| 2343 | const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_12 ), (int)((_MM_PERM_ENUM)160))); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) |
| 2344 | const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_13 ), (int)((_MM_PERM_ENUM)160))); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) |
| 2345 | const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_13 ), (int)((_MM_PERM_ENUM)160))); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) |
| 2346 | |
| 2347 | const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 2348 | const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)245))); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) |
| 2349 | const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 2350 | const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)245))); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) |
| 2351 | const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_02 ), (int)((_MM_PERM_ENUM)245))); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) |
| 2352 | const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_02 ), (int)((_MM_PERM_ENUM)245))); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) |
| 2353 | const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_03 ), (int)((_MM_PERM_ENUM)245))); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) |
| 2354 | const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_03 ), (int)((_MM_PERM_ENUM)245))); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) |
| 2355 | |
| 2356 | const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 2357 | const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)245))); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) |
| 2358 | const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 2359 | const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)245))); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) |
| 2360 | const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_12 ), (int)((_MM_PERM_ENUM)245))); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) |
| 2361 | const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_12 ), (int)((_MM_PERM_ENUM)245))); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) |
| 2362 | const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_13 ), (int)((_MM_PERM_ENUM)245))); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) |
| 2363 | const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_13 ), (int)((_MM_PERM_ENUM)245))); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) |
| 2364 | |
| 2365 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 2366 | __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1)); |
| 2367 | __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1)); |
| 2368 | __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1)); |
| 2369 | __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1)); |
| 2370 | __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1)); |
| 2371 | __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1)); |
| 2372 | __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1)); |
| 2373 | __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1)); |
| 2374 | |
| 2375 | __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2)); |
| 2376 | __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2)); |
| 2377 | __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2)); |
| 2378 | __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2)); |
| 2379 | __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2)); |
| 2380 | __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2)); |
| 2381 | __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2)); |
| 2382 | __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2)); |
| 2383 | |
| 2384 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 2385 | __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 2386 | __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 2387 | __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 2388 | __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 2389 | |
| 2390 | __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 2391 | __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 2392 | __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 2393 | __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 2394 | |
| 2395 | iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0); |
| 2396 | iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0); |
| 2397 | iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0); |
| 2398 | iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0); |
| 2399 | |
| 2400 | iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1); |
| 2401 | iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1); |
| 2402 | iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1); |
| 2403 | iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1); |
| 2404 | |
| 2405 | // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) |
| 2406 | __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01_0 ), (int)((_MM_PERM_ENUM)78)))); |
| 2407 | __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00_0 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01_0); |
| 2408 | __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11_0 ), (int)((_MM_PERM_ENUM)78)))); |
| 2409 | __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10_0 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11_0); |
| 2410 | __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01_1 ), (int)((_MM_PERM_ENUM)78)))); |
| 2411 | __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00_1 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01_1); |
| 2412 | __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11_1 ), (int)((_MM_PERM_ENUM)78)))); |
| 2413 | __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10_1 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11_1); |
| 2414 | |
| 2415 | __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1); |
| 2416 | __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1); |
| 2417 | __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1); |
| 2418 | __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1); |
| 2419 | |
| 2420 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 2421 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); |
| 2422 | const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 2423 | const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1)((__m512)__builtin_ia32_insertf32x8((__v16sf)(__m512)(_mm512_castps256_ps512 (row_scale_f32_ymm)), (__v8sf)(__m256)(row_scale_f32_ymm), (int )(1))); |
| 2424 | |
| 2425 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 2426 | acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 2427 | acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 2428 | acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 2429 | acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 2430 | |
| 2431 | __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 2432 | __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)85))), mins_01); |
| 2433 | __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 2434 | __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)255))), mins_01); |
| 2435 | |
| 2436 | acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_min_rows[rp * 4]); |
| 2437 | acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_min_rows[rp * 4 + 1]); |
| 2438 | acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_min_rows[rp * 4 + 2]); |
| 2439 | acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_min_rows[rp * 4 + 3]); |
| 2440 | } |
| 2441 | } |
| 2442 | } |
| 2443 | // Store the accumulated values |
| 2444 | for (int i = 0; i < 16; i++) { |
| 2445 | _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 2446 | } |
| 2447 | } |
| 2448 | } |
| 2449 | |
| 2450 | for (; y < nr / 4; y++) { |
| 2451 | |
| 2452 | const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); |
| 2453 | |
| 2454 | // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation |
| 2455 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 2456 | |
| 2457 | const block_q4_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 2458 | const block_q4_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 2459 | |
| 2460 | // Master FP accumulators |
| 2461 | __m512 acc_rows[4]; |
| 2462 | for (int i = 0; i < 4; i++) { |
| 2463 | acc_rows[i] = _mm512_setzero_ps(); |
| 2464 | } |
| 2465 | |
| 2466 | __m512 acc_min_rows[4]; |
| 2467 | for (int i = 0; i < 4; i++) { |
| 2468 | acc_min_rows[i] = _mm512_setzero_ps(); |
| 2469 | } |
| 2470 | |
| 2471 | // For super block |
| 2472 | for (int64_t b = 0; b < nb; b++) { |
| 2473 | // Scale values - Load the sixteen scale values from two block_q4_kx8 structures |
| 2474 | const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 2475 | |
| 2476 | // dmin values - Load the sixteen dmin values from two block_q4_kx8 structures |
| 2477 | const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin); |
| 2478 | |
| 2479 | // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration |
| 2480 | for (int sb = 0; sb < QK_K256 / 64; sb++) { |
| 2481 | |
| 2482 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256)); |
| 2483 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256)); |
| 2484 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256)); |
| 2485 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256)); |
| 2486 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256)); |
| 2487 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256)); |
| 2488 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256)); |
| 2489 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256)); |
| 2490 | |
| 2491 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256)); |
| 2492 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256)); |
| 2493 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256)); |
| 2494 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256)); |
| 2495 | const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256)); |
| 2496 | const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256)); |
| 2497 | const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256)); |
| 2498 | const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256)); |
| 2499 | |
| 2500 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 2501 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 2502 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 2503 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 2504 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 2505 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 2506 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 2507 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 2508 | |
| 2509 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 2510 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 2511 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 2512 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 2513 | const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2 , requiredOrder)), (int)(240))); |
| 2514 | const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_2 ), (int)(240))); |
| 2515 | const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3 , requiredOrder)), (int)(240))); |
| 2516 | const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_3 ), (int)(240))); |
| 2517 | |
| 2518 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 2519 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 2520 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 2521 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 2522 | |
| 2523 | const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_2)), (__v8si)(__m256i)(rhs_raw_mat_89CD_2), (int)(1))); |
| 2524 | const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_2)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_2), (int)(1))); |
| 2525 | const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_3)), (__v8si)(__m256i)(rhs_raw_mat_89CD_3), (int)(1))); |
| 2526 | const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_3)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_3), (int)(1))); |
| 2527 | |
| 2528 | //4-bit -> 8-bit |
| 2529 | const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0, m4bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7) |
| 2530 | const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0, m4bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7) |
| 2531 | const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1, m4bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15) |
| 2532 | const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1, m4bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15) |
| 2533 | |
| 2534 | const __m512i rhs_mat_014589CD_02 = _mm512_and_si512(rhs_raw_mat_014589CD_2, m4bexpanded); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) B08(16-23) B09(16-23) B0C(16-23) B0D(16-23) |
| 2535 | const __m512i rhs_mat_2367ABEF_02 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2, m4bexpanded); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) B0A(16-23) B0B(16-23) B0E(16-23) B0F(16-23) |
| 2536 | const __m512i rhs_mat_014589CD_03 = _mm512_and_si512(rhs_raw_mat_014589CD_3, m4bexpanded); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) B08(24-31) B09(24-31) B0C(24-31) B0D(24-31) |
| 2537 | const __m512i rhs_mat_2367ABEF_03 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3, m4bexpanded); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) B0A(24-31) B0B(24-31) B0E(24-31) B0F(24-31) |
| 2538 | |
| 2539 | const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m4bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7) |
| 2540 | const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m4bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7) |
| 2541 | const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m4bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15) |
| 2542 | const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m4bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15) |
| 2543 | |
| 2544 | const __m512i rhs_mat_014589CD_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m4bexpanded); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) B18(16-23) B19(16-23) B1C(16-23) B1D(16-23) |
| 2545 | const __m512i rhs_mat_2367ABEF_12 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m4bexpanded); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) B1A(16-23) B1B(16-23) B1E(16-23) B1F(16-23) |
| 2546 | const __m512i rhs_mat_014589CD_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m4bexpanded); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) B18(24-31) B19(24-31) B1C(24-31) B1D(24-31) |
| 2547 | const __m512i rhs_mat_2367ABEF_13 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m4bexpanded); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) B1A(24-31) B1B(24-31) B1E(24-31) B1F(24-31) |
| 2548 | |
| 2549 | // Shuffle pattern one - right side input |
| 2550 | const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3) |
| 2551 | const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3) |
| 2552 | const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 2553 | const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11) |
| 2554 | const __m512i rhs_mat_014589CD_02_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_02 ), (int)((_MM_PERM_ENUM)136))); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) B08(16-19) B09(16-19) B08(16-19) B09(16-19) B0C(16-19) B0D(16-19) B0C(16-19) B0D(16-19) |
| 2555 | const __m512i rhs_mat_2367ABEF_02_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_02 ), (int)((_MM_PERM_ENUM)136))); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) B0A(16-19) B0B(16-19) B0A(16-19) B0B(16-19) B0E(16-19) B0F(16-19) B0E(16-19) B0F(16-19) |
| 2556 | const __m512i rhs_mat_014589CD_03_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_03 ), (int)((_MM_PERM_ENUM)136))); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) B08(24-27) B09(24-27) B08(24-27) B09(24-27) B0C(24-27) B0D(24-27) B0C(24-27) B0D(24-27) |
| 2557 | const __m512i rhs_mat_2367ABEF_03_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_03 ), (int)((_MM_PERM_ENUM)136))); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) B0A(24-27) B0B(24-27) B0A(24-27) B0B(24-27) B0E(24-27) B0F(24-27) B0E(24-27) B0F(24-27) |
| 2558 | |
| 2559 | const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3) |
| 2560 | const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3) |
| 2561 | const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11) |
| 2562 | const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11) |
| 2563 | const __m512i rhs_mat_014589CD_12_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_12 ), (int)((_MM_PERM_ENUM)136))); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) B18(16-19) B19(16-19) B18(16-19) B19(16-19) B1C(16-19) B1D(16-19) B1C(16-19) B1D(16-19) |
| 2564 | const __m512i rhs_mat_2367ABEF_12_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_12 ), (int)((_MM_PERM_ENUM)136))); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) B1A(16-19) B1B(16-19) B1A(16-19) B1B(16-19) B1E(16-19) B1F(16-19) B1E(16-19) B1F(16-19) |
| 2565 | const __m512i rhs_mat_014589CD_13_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_13 ), (int)((_MM_PERM_ENUM)136))); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) B18(24-27) B19(24-27) B18(24-27) B19(24-27) B1C(24-27) B1D(24-27) B1C(24-27) B1D(24-27) |
| 2566 | const __m512i rhs_mat_2367ABEF_13_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_13 ), (int)((_MM_PERM_ENUM)136))); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) B1A(24-27) B1B(24-27) B1A(24-27) B1B(24-27) B1E(24-27) B1F(24-27) B1E(24-27) B1F(24-27) |
| 2567 | |
| 2568 | // Shuffle pattern two - right side input |
| 2569 | const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7) |
| 2570 | const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7) |
| 2571 | const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15) |
| 2572 | const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15) |
| 2573 | const __m512i rhs_mat_014589CD_02_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_02, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_02 ), (int)((_MM_PERM_ENUM)221))); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) B08(20-23) B09(20-23) B08(20-23) B09(20-23) B0C(20-23) B0D(20-23) B0C(20-23) B0D(20-23) |
| 2574 | const __m512i rhs_mat_2367ABEF_02_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_02, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_02 ), (int)((_MM_PERM_ENUM)221))); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) B0A(20-23) B0B(20-23) B0A(20-23) B0B(20-23) B0E(20-23) B0F(20-23) B0E(20-23) B0F(20-23) |
| 2575 | const __m512i rhs_mat_014589CD_03_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_03, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_03 ), (int)((_MM_PERM_ENUM)221))); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) B08(28-31) B09(28-31) B08(28-31) B09(28-31) B0C(28-31) B0D(28-31) B0C(28-31) 0BD(28-31) |
| 2576 | const __m512i rhs_mat_2367ABEF_03_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_03, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_03 ), (int)((_MM_PERM_ENUM)221))); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) B0A(28-31) B0B(28-31) B0A(28-31) B0B(28-31) B0E(28-31) B0F(28-31) B0E(28-31) B0F(28-31) |
| 2577 | |
| 2578 | const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7) |
| 2579 | const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7) |
| 2580 | const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15) |
| 2581 | const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15) |
| 2582 | const __m512i rhs_mat_014589CD_12_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_12, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_12 ), (int)((_MM_PERM_ENUM)221))); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) B18(20-23) B19(20-23) B18(20-23) B19(20-23) B1C(20-23) B1D(20-23) B1C(20-23) B1D(20-23) |
| 2583 | const __m512i rhs_mat_2367ABEF_12_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_12, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_12 ), (int)((_MM_PERM_ENUM)221))); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) B1A(20-23) B1B(20-23) B1A(20-23) B1B(20-23) B1E(20-23) B1F(20-23) B1E(20-23) B1F(20-23) |
| 2584 | const __m512i rhs_mat_014589CD_13_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_13, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_13 ), (int)((_MM_PERM_ENUM)221))); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) B18(28-31) B19(28-31) B18(28-31) B19(28-31) B1C(28-31) B1D(28-31) B1C(28-31) B1D(28-31) |
| 2585 | const __m512i rhs_mat_2367ABEF_13_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_13, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_13 ), (int)((_MM_PERM_ENUM)221))); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) B1A(28-31) B1B(28-31) B1A(28-31) B1B(28-31) B1E(28-31) B1F(28-31) B1E(28-31) B1F(28-31) |
| 2586 | |
| 2587 | uint32_t utmp_00[4], utmp_01[4], utmp_10[4], utmp_11[4]; |
| 2588 | |
| 2589 | // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together |
| 2590 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 2591 | memcpy(utmp_00, b_ptr_0[b].scales + 24 * sb, 12); |
| 2592 | utmp_00[3] = ((utmp_00[2] >> 4) & kmask2) | (((utmp_00[1] >> 6) & kmask3) << 4); |
| 2593 | const uint32_t uaux_00 = utmp_00[1] & kmask1; |
| 2594 | utmp_00[1] = (utmp_00[2] & kmask2) | (((utmp_00[0] >> 6) & kmask3) << 4); |
| 2595 | utmp_00[2] = uaux_00; |
| 2596 | utmp_00[0] &= kmask1; |
| 2597 | |
| 2598 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 2599 | memcpy(utmp_01, b_ptr_0[b].scales + 12 + sb * 24, 12); |
| 2600 | utmp_01[3] = ((utmp_01[2] >> 4) & kmask2) | (((utmp_01[1] >> 6) & kmask3) << 4); |
| 2601 | const uint32_t uaux_01 = utmp_01[1] & kmask1; |
| 2602 | utmp_01[1] = (utmp_01[2] & kmask2) | (((utmp_01[0] >> 6) & kmask3) << 4); |
| 2603 | utmp_01[2] = uaux_01; |
| 2604 | utmp_01[0] &= kmask1; |
| 2605 | |
| 2606 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 2607 | memcpy(utmp_10, b_ptr_1[b].scales + sb * 24, 12); |
| 2608 | utmp_10[3] = ((utmp_10[2] >> 4) & kmask2) | (((utmp_10[1] >> 6) & kmask3) << 4); |
| 2609 | const uint32_t uaux_10 = utmp_10[1] & kmask1; |
| 2610 | utmp_10[1] = (utmp_10[2] & kmask2) | (((utmp_10[0] >> 6) & kmask3) << 4); |
| 2611 | utmp_10[2] = uaux_10; |
| 2612 | utmp_10[0] &= kmask1; |
| 2613 | |
| 2614 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 2615 | memcpy(utmp_11, b_ptr_1[b].scales + 12 + sb * 24, 12); |
| 2616 | utmp_11[3] = ((utmp_11[2] >> 4) & kmask2) | (((utmp_11[1] >> 6) & kmask3) << 4); |
| 2617 | const uint32_t uaux_11 = utmp_11[1] & kmask1; |
| 2618 | utmp_11[1] = (utmp_11[2] & kmask2) | (((utmp_11[0] >> 6) & kmask3) << 4); |
| 2619 | utmp_11[2] = uaux_11; |
| 2620 | utmp_11[0] &= kmask1; |
| 2621 | |
| 2622 | // Scales of first sub block in the sb loop |
| 2623 | const __m256i mins_and_scales_0 = _mm256_set_epi32(utmp_10[3], utmp_10[2], utmp_10[1], utmp_10[0], utmp_00[3], utmp_00[2], utmp_00[1], utmp_00[0]); |
| 2624 | const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); |
| 2625 | |
| 2626 | // Scales of second sub block in the sb loop |
| 2627 | const __m256i mins_and_scales_1 = _mm256_set_epi32(utmp_11[3], utmp_11[2], utmp_11[1], utmp_11[0], utmp_01[3], utmp_01[2], utmp_01[1], utmp_01[0]); |
| 2628 | const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); |
| 2629 | |
| 2630 | // Mins of first and second sub block of Q4_K block are arranged side by side |
| 2631 | const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_unpacklo_epi8(_mm256_shuffle_epi32(mins_and_scales_0, 78)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(mins_and_scales_0 ), (int)(78))), _mm256_shuffle_epi32(mins_and_scales_1, 78)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(mins_and_scales_1 ), (int)(78))))); |
| 2632 | |
| 2633 | const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)68))); |
| 2634 | const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)238))); |
| 2635 | |
| 2636 | const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)68))); |
| 2637 | const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)238))); |
| 2638 | |
| 2639 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 2640 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 2641 | __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb))); |
| 2642 | __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(0))); |
| 2643 | __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(17))); |
| 2644 | __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb))); |
| 2645 | __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(0))); |
| 2646 | __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(17))); |
| 2647 | __m256i lhs_mat_ymm_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb))); |
| 2648 | __m256i lhs_mat_ymm_01_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_02 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_02), (int)(0))); |
| 2649 | __m256i lhs_mat_ymm_23_02 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_02, lhs_mat_ymm_0123_02, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_02 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_02), (int)(17))); |
| 2650 | __m256i lhs_mat_ymm_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb))); |
| 2651 | __m256i lhs_mat_ymm_01_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_03 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_03), (int)(0))); |
| 2652 | __m256i lhs_mat_ymm_23_03 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_03, lhs_mat_ymm_0123_03, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_03 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_03), (int)(17))); |
| 2653 | __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb))); |
| 2654 | __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(0))); |
| 2655 | __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(17))); |
| 2656 | __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb))); |
| 2657 | __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(0))); |
| 2658 | __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(17))); |
| 2659 | __m256i lhs_mat_ymm_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb))); |
| 2660 | __m256i lhs_mat_ymm_01_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_12 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_12), (int)(0))); |
| 2661 | __m256i lhs_mat_ymm_23_12 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_12, lhs_mat_ymm_0123_12, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_12 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_12), (int)(17))); |
| 2662 | __m256i lhs_mat_ymm_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb))); |
| 2663 | __m256i lhs_mat_ymm_01_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_13 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_13), (int)(0))); |
| 2664 | __m256i lhs_mat_ymm_23_13 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_13, lhs_mat_ymm_0123_13, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_13 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_13), (int)(17))); |
| 2665 | |
| 2666 | //Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into a 512 bit vector |
| 2667 | __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_00)), (__v8si)(__m256i)(lhs_mat_ymm_01_00), ( int)(1))); |
| 2668 | __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_00)), (__v8si)(__m256i)(lhs_mat_ymm_23_00), ( int)(1))); |
| 2669 | __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_01)), (__v8si)(__m256i)(lhs_mat_ymm_01_01), ( int)(1))); |
| 2670 | __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_01)), (__v8si)(__m256i)(lhs_mat_ymm_23_01), ( int)(1))); |
| 2671 | __m512i lhs_mat_01_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_02), lhs_mat_ymm_01_02, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_02)), (__v8si)(__m256i)(lhs_mat_ymm_01_02), ( int)(1))); |
| 2672 | __m512i lhs_mat_23_02 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_02), lhs_mat_ymm_23_02, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_02)), (__v8si)(__m256i)(lhs_mat_ymm_23_02), ( int)(1))); |
| 2673 | __m512i lhs_mat_01_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_03), lhs_mat_ymm_01_03, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_03)), (__v8si)(__m256i)(lhs_mat_ymm_01_03), ( int)(1))); |
| 2674 | __m512i lhs_mat_23_03 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_03), lhs_mat_ymm_23_03, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_03)), (__v8si)(__m256i)(lhs_mat_ymm_23_03), ( int)(1))); |
| 2675 | |
| 2676 | __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_10)), (__v8si)(__m256i)(lhs_mat_ymm_01_10), ( int)(1))); |
| 2677 | __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_10)), (__v8si)(__m256i)(lhs_mat_ymm_23_10), ( int)(1))); |
| 2678 | __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_11)), (__v8si)(__m256i)(lhs_mat_ymm_01_11), ( int)(1))); |
| 2679 | __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_11)), (__v8si)(__m256i)(lhs_mat_ymm_23_11), ( int)(1))); |
| 2680 | __m512i lhs_mat_01_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_12), lhs_mat_ymm_01_12, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_12)), (__v8si)(__m256i)(lhs_mat_ymm_01_12), ( int)(1))); |
| 2681 | __m512i lhs_mat_23_12 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_12), lhs_mat_ymm_23_12, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_12)), (__v8si)(__m256i)(lhs_mat_ymm_23_12), ( int)(1))); |
| 2682 | __m512i lhs_mat_01_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_13), lhs_mat_ymm_01_13, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_13)), (__v8si)(__m256i)(lhs_mat_ymm_01_13), ( int)(1))); |
| 2683 | __m512i lhs_mat_23_13 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_13), lhs_mat_ymm_23_13, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_13)), (__v8si)(__m256i)(lhs_mat_ymm_23_13), ( int)(1))); |
| 2684 | |
| 2685 | // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks |
| 2686 | __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb))); |
| 2687 | __m256i lhs_bsums_hsum_ymm_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (lhs_bsums_0123_01), (int)(1))))); |
| 2688 | lhs_bsums_hsum_ymm_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_ymm_0123_01, lhs_bsums_hsum_ymm_0123_01, 0)((__m256i)__builtin_ia32_permti256((__m256i)(lhs_bsums_hsum_ymm_0123_01 ), (__m256i)(lhs_bsums_hsum_ymm_0123_01), (int)(0))); |
| 2689 | __m512i lhs_bsums_hsum_0123_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_hsum_ymm_0123_01), lhs_bsums_hsum_ymm_0123_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_hsum_ymm_0123_01)), (__v8si)(__m256i)(lhs_bsums_hsum_ymm_0123_01 ), (int)(1))); |
| 2690 | |
| 2691 | // Shuffle pattern one - left side input |
| 2692 | const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 2693 | const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)160))); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) |
| 2694 | const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 2695 | const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)160))); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) |
| 2696 | const __m512i lhs_mat_01_02_sp1 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_02 ), (int)((_MM_PERM_ENUM)160))); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) |
| 2697 | const __m512i lhs_mat_23_02_sp1 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_02 ), (int)((_MM_PERM_ENUM)160))); //A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) A02(16-19) A02(16-19) A03(16-19) A03(16-19) |
| 2698 | const __m512i lhs_mat_01_03_sp1 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_03 ), (int)((_MM_PERM_ENUM)160))); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) |
| 2699 | const __m512i lhs_mat_23_03_sp1 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_03 ), (int)((_MM_PERM_ENUM)160))); //A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) A02(24-27) A02(24-27) A03(24-27) A03(24-27) |
| 2700 | |
| 2701 | const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 2702 | const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)160))); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) |
| 2703 | const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 2704 | const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)160))); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) |
| 2705 | const __m512i lhs_mat_01_12_sp1 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_12 ), (int)((_MM_PERM_ENUM)160))); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) |
| 2706 | const __m512i lhs_mat_23_12_sp1 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_12 ), (int)((_MM_PERM_ENUM)160))); //A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) A12(16-19) A12(16-19) A13(16-19) A13(16-19) |
| 2707 | const __m512i lhs_mat_01_13_sp1 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_13 ), (int)((_MM_PERM_ENUM)160))); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) |
| 2708 | const __m512i lhs_mat_23_13_sp1 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_13 ), (int)((_MM_PERM_ENUM)160))); //A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) A12(24-27) A12(24-27) A13(24-27) A13(24-27) |
| 2709 | |
| 2710 | const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 2711 | const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)245))); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) |
| 2712 | const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 2713 | const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)245))); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) |
| 2714 | const __m512i lhs_mat_01_02_sp2 = _mm512_shuffle_epi32(lhs_mat_01_02, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_02 ), (int)((_MM_PERM_ENUM)245))); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) |
| 2715 | const __m512i lhs_mat_23_02_sp2 = _mm512_shuffle_epi32(lhs_mat_23_02, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_02 ), (int)((_MM_PERM_ENUM)245))); //A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) A02(20-23) A02(20-23) A03(20-23) A03(20-23) |
| 2716 | const __m512i lhs_mat_01_03_sp2 = _mm512_shuffle_epi32(lhs_mat_01_03, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_03 ), (int)((_MM_PERM_ENUM)245))); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) |
| 2717 | const __m512i lhs_mat_23_03_sp2 = _mm512_shuffle_epi32(lhs_mat_23_03, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_03 ), (int)((_MM_PERM_ENUM)245))); //A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) A02(28-31) A02(28-31) A03(28-31) A03(28-31) |
| 2718 | |
| 2719 | const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 2720 | const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)245))); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) |
| 2721 | const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 2722 | const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)245))); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) |
| 2723 | const __m512i lhs_mat_01_12_sp2 = _mm512_shuffle_epi32(lhs_mat_01_12, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_12 ), (int)((_MM_PERM_ENUM)245))); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) |
| 2724 | const __m512i lhs_mat_23_12_sp2 = _mm512_shuffle_epi32(lhs_mat_23_12, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_12 ), (int)((_MM_PERM_ENUM)245))); //A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) A12(20-23) A12(20-23) A13(20-23) A13(20-23) |
| 2725 | const __m512i lhs_mat_01_13_sp2 = _mm512_shuffle_epi32(lhs_mat_01_13, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_13 ), (int)((_MM_PERM_ENUM)245))); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) |
| 2726 | const __m512i lhs_mat_23_13_sp2 = _mm512_shuffle_epi32(lhs_mat_23_13, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_13 ), (int)((_MM_PERM_ENUM)245))); //A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) A12(28-31) A12(28-31) A13(28-31) A13(28-31) |
| 2727 | |
| 2728 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 2729 | __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1)); |
| 2730 | __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_01_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_01_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1)); |
| 2731 | __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1)); |
| 2732 | __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp1, lhs_mat_23_03_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp1, lhs_mat_23_02_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1)); |
| 2733 | __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1)); |
| 2734 | __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_01_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_01_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1)); |
| 2735 | __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1)); |
| 2736 | __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp1, lhs_mat_23_13_sp1), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp1, lhs_mat_23_12_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1)); |
| 2737 | |
| 2738 | __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2)); |
| 2739 | __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_01_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_01_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2)); |
| 2740 | __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2)); |
| 2741 | __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_03_sp2, lhs_mat_23_03_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_02_sp2, lhs_mat_23_02_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2)); |
| 2742 | __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2)); |
| 2743 | __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_01_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_01_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2)); |
| 2744 | __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_014589CD_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2)); |
| 2745 | __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_add_epi16(_mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_13_sp2, lhs_mat_23_13_sp2), _mm512_maddubs_epi16(rhs_mat_2367ABEF_12_sp2, lhs_mat_23_12_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)), _mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2)); |
| 2746 | |
| 2747 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 2748 | __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 2749 | __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 2750 | __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 2751 | __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 2752 | |
| 2753 | __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 2754 | __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 2755 | __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 2756 | __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 2757 | |
| 2758 | iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0); |
| 2759 | iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0); |
| 2760 | iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0); |
| 2761 | iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0); |
| 2762 | |
| 2763 | iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1); |
| 2764 | iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1); |
| 2765 | iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1); |
| 2766 | iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1); |
| 2767 | |
| 2768 | // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) |
| 2769 | __m512i iacc_row_0_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_0, _mm512_shuffle_epi32(iacc_mat_01_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01_0 ), (int)((_MM_PERM_ENUM)78)))); |
| 2770 | __m512i iacc_row_1_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00_0 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01_0); |
| 2771 | __m512i iacc_row_2_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_0, _mm512_shuffle_epi32(iacc_mat_11_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11_0 ), (int)((_MM_PERM_ENUM)78)))); |
| 2772 | __m512i iacc_row_3_0 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10_0, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10_0 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11_0); |
| 2773 | __m512i iacc_row_0_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00_1, _mm512_shuffle_epi32(iacc_mat_01_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01_1 ), (int)((_MM_PERM_ENUM)78)))); |
| 2774 | __m512i iacc_row_1_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00_1 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01_1); |
| 2775 | __m512i iacc_row_2_1 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10_1, _mm512_shuffle_epi32(iacc_mat_11_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11_1 ), (int)((_MM_PERM_ENUM)78)))); |
| 2776 | __m512i iacc_row_3_1 = _mm512_mask_blend_epi32(0xCCCC,_mm512_shuffle_epi32(iacc_mat_10_1, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10_1 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11_1); |
| 2777 | |
| 2778 | __m512i iacc_row_0 = _mm512_add_epi32(iacc_row_0_0, iacc_row_0_1); |
| 2779 | __m512i iacc_row_1 = _mm512_add_epi32(iacc_row_1_0, iacc_row_1_1); |
| 2780 | __m512i iacc_row_2 = _mm512_add_epi32(iacc_row_2_0, iacc_row_2_1); |
| 2781 | __m512i iacc_row_3 = _mm512_add_epi32(iacc_row_3_0, iacc_row_3_1); |
| 2782 | |
| 2783 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 2784 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); |
| 2785 | const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 2786 | const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1)((__m512)__builtin_ia32_insertf32x8((__v16sf)(__m512)(_mm512_castps256_ps512 (row_scale_f32_ymm)), (__v8sf)(__m256)(row_scale_f32_ymm), (int )(1))); |
| 2787 | |
| 2788 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 2789 | acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 2790 | acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 2791 | acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 2792 | acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 2793 | |
| 2794 | __m512i iacc_row_min_0 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 2795 | __m512i iacc_row_min_1 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)85))), mins_01); |
| 2796 | __m512i iacc_row_min_2 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 2797 | __m512i iacc_row_min_3 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_hsum_0123_01, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_hsum_0123_01 ), (int)((_MM_PERM_ENUM)255))), mins_01); |
| 2798 | |
| 2799 | acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_min_rows[0]); |
| 2800 | acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_min_rows[1]); |
| 2801 | acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_min_rows[2]); |
| 2802 | acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_min_rows[3]); |
| 2803 | } |
| 2804 | } |
| 2805 | // Store accumulated values |
| 2806 | for (int i = 0; i < 4; i++) { |
| 2807 | _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 2808 | } |
| 2809 | } |
| 2810 | } |
| 2811 | if (anc != nc) { |
| 2812 | xstart = anc/8; |
| 2813 | y = 0; |
| 2814 | } |
| 2815 | #endif // __AVX512BW__ && __AVX512DQ__ |
| 2816 | |
| 2817 | // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation |
| 2818 | for (; y < anr / 4; y += 4) { |
| 2819 | |
| 2820 | const block_q8_Kx4 * a_ptrs[4]; |
| 2821 | |
| 2822 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 2823 | for (int i = 0; i < 3; ++i) { |
| 2824 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 2825 | } |
| 2826 | |
| 2827 | // Take group of eight block_q4_kx8 structures at each pass of the loop and perform dot product operation |
| 2828 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 2829 | |
| 2830 | const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 2831 | |
| 2832 | // Master FP accumulators |
| 2833 | __m256 acc_rows[16]; |
| 2834 | for (int i = 0; i < 16; i++) { |
| 2835 | acc_rows[i] = _mm256_setzero_ps(); |
| 2836 | } |
| 2837 | |
| 2838 | __m256 acc_min_rows[16]; |
| 2839 | for (int i = 0; i < 16; i++) { |
| 2840 | acc_min_rows[i] = _mm256_setzero_ps(); |
| 2841 | } |
| 2842 | |
| 2843 | // For super block |
| 2844 | for (int64_t b = 0; b < nb; b++) { |
| 2845 | |
| 2846 | // Scale values - Load the eight scale values of block_q4_kx8 |
| 2847 | const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 2848 | |
| 2849 | // dmin values - Load the eight dmin values of block_q4_kx8 |
| 2850 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 2851 | |
| 2852 | // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration |
| 2853 | for (int sb = 0; sb < QK_K256 / 64; sb++) { |
| 2854 | |
| 2855 | // Load the eight block_q4_K for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 2856 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); |
| 2857 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); |
| 2858 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); |
| 2859 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); |
| 2860 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); |
| 2861 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); |
| 2862 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); |
| 2863 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); |
| 2864 | |
| 2865 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 2866 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 2867 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 2868 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 2869 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 2870 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 2871 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 2872 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 2873 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 2874 | |
| 2875 | // 4-bit -> 8-bit |
| 2876 | // First sub block of the two sub blocks processed in the iteration |
| 2877 | const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) |
| 2878 | const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) |
| 2879 | |
| 2880 | const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) |
| 2881 | const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) |
| 2882 | |
| 2883 | const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) |
| 2884 | const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) |
| 2885 | |
| 2886 | const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) |
| 2887 | const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) |
| 2888 | |
| 2889 | // Second sub block of the two sub blocks processed in the iteration |
| 2890 | const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) |
| 2891 | const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) |
| 2892 | |
| 2893 | const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) |
| 2894 | const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) |
| 2895 | |
| 2896 | const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) |
| 2897 | const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) |
| 2898 | |
| 2899 | const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) |
| 2900 | const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) |
| 2901 | |
| 2902 | // Shuffle pattern one - right side input |
| 2903 | const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) |
| 2904 | const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) |
| 2905 | |
| 2906 | const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) |
| 2907 | const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) |
| 2908 | |
| 2909 | const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_02 ), (int)(136))); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) |
| 2910 | const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_02 ), (int)(136))); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) |
| 2911 | |
| 2912 | const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_03 ), (int)(136))); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) |
| 2913 | const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_03 ), (int)(136))); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) |
| 2914 | |
| 2915 | const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) |
| 2916 | const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) |
| 2917 | |
| 2918 | const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) |
| 2919 | const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) |
| 2920 | |
| 2921 | const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_12 ), (int)(136))); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) |
| 2922 | const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_12 ), (int)(136))); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) |
| 2923 | |
| 2924 | const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_13 ), (int)(136))); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) |
| 2925 | const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_13 ), (int)(136))); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) |
| 2926 | |
| 2927 | |
| 2928 | // Shuffle pattern two - right side input |
| 2929 | const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) |
| 2930 | const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) |
| 2931 | |
| 2932 | const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) |
| 2933 | const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) |
| 2934 | |
| 2935 | const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_02 ), (int)(221))); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) |
| 2936 | const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_02 ), (int)(221))); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) |
| 2937 | |
| 2938 | const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_03 ), (int)(221))); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) |
| 2939 | const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_03 ), (int)(221))); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) |
| 2940 | |
| 2941 | const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) |
| 2942 | const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) |
| 2943 | |
| 2944 | const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) |
| 2945 | const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) |
| 2946 | |
| 2947 | const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_12 ), (int)(221))); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) |
| 2948 | const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_12 ), (int)(221))); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) |
| 2949 | |
| 2950 | const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_13 ), (int)(221))); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) |
| 2951 | const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_13 ), (int)(221))); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) |
| 2952 | |
| 2953 | uint32_t utmp_0[4], utmp_1[4]; |
| 2954 | |
| 2955 | // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together |
| 2956 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 2957 | memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); |
| 2958 | utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); |
| 2959 | const uint32_t uaux_0 = utmp_0[1] & kmask1; |
| 2960 | utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); |
| 2961 | utmp_0[2] = uaux_0; |
| 2962 | utmp_0[0] &= kmask1; |
| 2963 | |
| 2964 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures for the sb loop |
| 2965 | memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); |
| 2966 | utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); |
| 2967 | const uint32_t uaux_1 = utmp_1[1] & kmask1; |
| 2968 | utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); |
| 2969 | utmp_1[2] = uaux_1; |
| 2970 | utmp_1[0] &= kmask1; |
| 2971 | |
| 2972 | // Scales of first sub block in the sb loop |
| 2973 | const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); |
| 2974 | const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); |
| 2975 | |
| 2976 | // Scales of second sub block in the sb loop |
| 2977 | const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); |
| 2978 | const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); |
| 2979 | |
| 2980 | // Mins of first and second sub block of Q4_K block are arranged side by side |
| 2981 | const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_0 ), (int)(78))), _mm_shuffle_epi32(mins_and_scales_1, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_1 ), (int)(78))))); |
| 2982 | |
| 2983 | const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(68))); |
| 2984 | const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(238))); |
| 2985 | |
| 2986 | const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(68))); |
| 2987 | const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(238))); |
| 2988 | |
| 2989 | for (int rp = 0; rp < 4; rp++) { |
| 2990 | |
| 2991 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 2992 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 2993 | __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 * sb))); |
| 2994 | __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(0))); |
| 2995 | __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(17))); |
| 2996 | __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 256 * sb))); |
| 2997 | __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(0))); |
| 2998 | __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(17))); |
| 2999 | __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 256 * sb))); |
| 3000 | __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_02 ), (__v8si)(__m256i)(lhs_mat_0123_02), (int)(0))); |
| 3001 | __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_02 ), (__v8si)(__m256i)(lhs_mat_0123_02), (int)(17))); |
| 3002 | __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 256 * sb))); |
| 3003 | __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_03 ), (__v8si)(__m256i)(lhs_mat_0123_03), (int)(0))); |
| 3004 | __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_03 ), (__v8si)(__m256i)(lhs_mat_0123_03), (int)(17))); |
| 3005 | __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 256 * sb))); |
| 3006 | __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(0))); |
| 3007 | __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(17))); |
| 3008 | __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 256 * sb))); |
| 3009 | __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(0))); |
| 3010 | __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(17))); |
| 3011 | __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 256 * sb))); |
| 3012 | __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_12 ), (__v8si)(__m256i)(lhs_mat_0123_12), (int)(0))); |
| 3013 | __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_12 ), (__v8si)(__m256i)(lhs_mat_0123_12), (int)(17))); |
| 3014 | __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 256 * sb))); |
| 3015 | __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_13 ), (__v8si)(__m256i)(lhs_mat_0123_13), (int)(0))); |
| 3016 | __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_13 ), (__v8si)(__m256i)(lhs_mat_0123_13), (int)(17))); |
| 3017 | |
| 3018 | // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks |
| 3019 | __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].bsums + 16 * sb))); |
| 3020 | __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (lhs_bsums_0123_01), (int)(1))))); |
| 3021 | lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0)((__m256i)__builtin_ia32_permti256((__m256i)(lhs_bsums_hsum_0123_01 ), (__m256i)(lhs_bsums_hsum_0123_01), (int)(0))); |
| 3022 | |
| 3023 | // Shuffle pattern one - left side input |
| 3024 | const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 3025 | const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(160))); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) |
| 3026 | |
| 3027 | const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 3028 | const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(160))); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) |
| 3029 | |
| 3030 | const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_02 ), (int)(160))); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) |
| 3031 | const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_02 ), (int)(160))); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) |
| 3032 | |
| 3033 | const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_03 ), (int)(160))); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) |
| 3034 | const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_03 ), (int)(160))); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) |
| 3035 | |
| 3036 | const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 3037 | const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(160))); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) |
| 3038 | |
| 3039 | const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 3040 | const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(160))); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) |
| 3041 | |
| 3042 | const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_12 ), (int)(160))); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) |
| 3043 | const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_12 ), (int)(160))); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) |
| 3044 | |
| 3045 | const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_13 ), (int)(160))); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) |
| 3046 | const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_13 ), (int)(160))); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) |
| 3047 | |
| 3048 | // Shuffle pattern two- left side input |
| 3049 | const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 3050 | const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(245))); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) |
| 3051 | |
| 3052 | const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 3053 | const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(245))); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) |
| 3054 | |
| 3055 | const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_02 ), (int)(245))); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) |
| 3056 | const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_02 ), (int)(245))); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) |
| 3057 | |
| 3058 | const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_03 ), (int)(245))); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) |
| 3059 | const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_03 ), (int)(245))); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) |
| 3060 | |
| 3061 | const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 3062 | const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(245))); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) |
| 3063 | |
| 3064 | const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 3065 | const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(245))); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) |
| 3066 | |
| 3067 | const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_12 ), (int)(245))); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) |
| 3068 | const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_12 ), (int)(245))); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) |
| 3069 | |
| 3070 | const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_13 ), (int)(245))); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) |
| 3071 | const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_13 ), (int)(245))); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) |
| 3072 | |
| 3073 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 3074 | __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); |
| 3075 | __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); |
| 3076 | __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); |
| 3077 | __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); |
| 3078 | __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); |
| 3079 | __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); |
| 3080 | __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); |
| 3081 | __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); |
| 3082 | |
| 3083 | __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); |
| 3084 | __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); |
| 3085 | __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); |
| 3086 | __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); |
| 3087 | __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); |
| 3088 | __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); |
| 3089 | __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); |
| 3090 | __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); |
| 3091 | |
| 3092 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 3093 | __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 3094 | __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 3095 | __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 3096 | __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 3097 | |
| 3098 | __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 3099 | __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 3100 | __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 3101 | __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 3102 | |
| 3103 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 3104 | iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); |
| 3105 | iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); |
| 3106 | iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); |
| 3107 | iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); |
| 3108 | |
| 3109 | iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); |
| 3110 | iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); |
| 3111 | iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); |
| 3112 | iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); |
| 3113 | |
| 3114 | // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) |
| 3115 | __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00_0 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01_0), (int)(78)))), (int)(204))); |
| 3116 | __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00_0), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_01_0), (int)(204))); |
| 3117 | __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10_0 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11_0), (int)(78)))), (int)(204))); |
| 3118 | __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10_0), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_11_0), (int)(204))); |
| 3119 | __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00_1 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01_1), (int)(78)))), (int)(204))); |
| 3120 | __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00_1), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_01_1), (int)(204))); |
| 3121 | __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10_1 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11_1), (int)(78)))), (int)(204))); |
| 3122 | __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10_1), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_11_1), (int)(204))); |
| 3123 | |
| 3124 | __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); |
| 3125 | __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); |
| 3126 | __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); |
| 3127 | __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); |
| 3128 | |
| 3129 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 3130 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); |
| 3131 | const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse);//GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); |
| 3132 | |
| 3133 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 3134 | acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 3135 | acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 3136 | acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 3137 | acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 3138 | |
| 3139 | __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(0))), mins_01); |
| 3140 | __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(85))), mins_01); |
| 3141 | __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(170))), mins_01); |
| 3142 | __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(255))), mins_01); |
| 3143 | |
| 3144 | acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_min_rows[rp * 4]); |
| 3145 | acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_min_rows[rp * 4 + 1]); |
| 3146 | acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_min_rows[rp * 4 + 2]); |
| 3147 | acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_min_rows[rp * 4 + 3]); |
| 3148 | |
| 3149 | } |
| 3150 | } |
| 3151 | } |
| 3152 | // Store the accumulated values |
| 3153 | for (int i = 0; i < 16; i++) { |
| 3154 | _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 3155 | } |
| 3156 | } |
| 3157 | } |
| 3158 | for (; y < nr / 4; y++) { |
| 3159 | |
| 3160 | const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); |
| 3161 | |
| 3162 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 3163 | |
| 3164 | const block_q4_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 3165 | |
| 3166 | // Master FP accumulators |
| 3167 | __m256 acc_rows[4]; |
| 3168 | for (int i = 0; i < 4; i++) { |
| 3169 | acc_rows[i] = _mm256_setzero_ps(); |
| 3170 | } |
| 3171 | |
| 3172 | __m256 acc_min_rows[4]; |
| 3173 | for (int i = 0; i < 4; i++) { |
| 3174 | acc_min_rows[i] = _mm256_setzero_ps(); |
| 3175 | } |
| 3176 | |
| 3177 | for (int64_t b = 0; b < nb; b++) { |
| 3178 | |
| 3179 | // Scale values - Load the eight scale values of block_q4_Kx8 |
| 3180 | const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 3181 | |
| 3182 | // dmin values - Load the eight dmin values of block_q4_Kx8 |
| 3183 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 3184 | |
| 3185 | // Loop to iterate over the eight sub blocks of a super block - two sub blocks are processed per iteration |
| 3186 | for (int sb = 0; sb < QK_K256 / 64; sb++) { |
| 3187 | |
| 3188 | // Load the eight block_q4_k for two sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 3189 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + sb * 256)); |
| 3190 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 32 + sb * 256)); |
| 3191 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 64 + sb * 256)); |
| 3192 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 96 + sb * 256)); |
| 3193 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 128 + sb * 256)); |
| 3194 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 160 + sb * 256)); |
| 3195 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 192 + sb * 256)); |
| 3196 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr[b].qs + 224 + sb * 256)); |
| 3197 | |
| 3198 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 3199 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 3200 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 3201 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 3202 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 3203 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 3204 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 3205 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 3206 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 3207 | |
| 3208 | // 4-bit -> 8-bit |
| 3209 | // First sub block of the two sub blocks processed in the iteration |
| 3210 | const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m4b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) |
| 3211 | const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m4b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) |
| 3212 | |
| 3213 | const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m4b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) |
| 3214 | const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m4b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) |
| 3215 | |
| 3216 | const __m256i rhs_mat_0145_02 = _mm256_and_si256(rhs_raw_mat_0145_2, m4b); //B00(16-23) B01(16-23) B04(16-23) B05(16-23) |
| 3217 | const __m256i rhs_mat_2367_02 = _mm256_and_si256(rhs_raw_mat_2367_2, m4b); //B02(16-23) B03(16-23) B06(16-23) B07(16-23) |
| 3218 | |
| 3219 | const __m256i rhs_mat_0145_03 = _mm256_and_si256(rhs_raw_mat_0145_3, m4b); //B00(24-31) B01(24-31) B04(24-31) B05(24-31) |
| 3220 | const __m256i rhs_mat_2367_03 = _mm256_and_si256(rhs_raw_mat_2367_3, m4b); //B02(24-31) B03(24-31) B06(24-31) B07(24-31) |
| 3221 | |
| 3222 | // Second sub block of the two sub blocks processed in the iteration |
| 3223 | const __m256i rhs_mat_0145_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m4b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) |
| 3224 | const __m256i rhs_mat_2367_10 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m4b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) |
| 3225 | |
| 3226 | const __m256i rhs_mat_0145_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m4b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) |
| 3227 | const __m256i rhs_mat_2367_11 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m4b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) |
| 3228 | |
| 3229 | const __m256i rhs_mat_0145_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m4b); //B10(16-23) B11(16-23) B14(16-23) B15(16-23) |
| 3230 | const __m256i rhs_mat_2367_12 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m4b); //B12(16-23) B13(16-23) B16(16-23) B17(16-23) |
| 3231 | |
| 3232 | const __m256i rhs_mat_0145_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m4b); //B10(24-31) B11(24-31) B14(24-31) B15(24-31) |
| 3233 | const __m256i rhs_mat_2367_13 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m4b); //B12(24-31) B13(24-31) B16(24-31) B17(24-31) |
| 3234 | |
| 3235 | // Shuffle pattern one - right side input |
| 3236 | const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) |
| 3237 | const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) |
| 3238 | |
| 3239 | const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) |
| 3240 | const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) |
| 3241 | |
| 3242 | const __m256i rhs_mat_0145_02_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_02, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_02 ), (int)(136))); //B00(16-19) B01(16-19) B00(16-19) B01(16-19) B04(16-19) B05(16-19) B04(16-19) B05(16-19) |
| 3243 | const __m256i rhs_mat_2367_02_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_02, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_02 ), (int)(136))); //B02(16-19) B03(16-19) B02(16-19) B03(16-19) B06(16-19) B07(16-19) B06(16-19) B07(16-19) |
| 3244 | |
| 3245 | const __m256i rhs_mat_0145_03_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_03, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_03 ), (int)(136))); //B00(24-27) B01(24-27) B00(24-27) B01(24-27) B04(24-27) B05(24-27) B04(24-27) B05(24-27) |
| 3246 | const __m256i rhs_mat_2367_03_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_03, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_03 ), (int)(136))); //B02(24-27) B03(24-27) B02(24-27) B03(24-27) B06(24-27) B07(24-27) B06(24-27) B07(24-27) |
| 3247 | |
| 3248 | const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) |
| 3249 | const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) |
| 3250 | |
| 3251 | const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) |
| 3252 | const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) |
| 3253 | |
| 3254 | const __m256i rhs_mat_0145_12_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_12, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_12 ), (int)(136))); //B10(16-19) B11(16-19) B10(16-19) B11(16-19) B14(16-19) B15(16-19) B14(16-19) B15(16-19) |
| 3255 | const __m256i rhs_mat_2367_12_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_12, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_12 ), (int)(136))); //B12(16-19) B13(16-19) B12(16-19) B13(16-19) B16(16-19) B17(16-19) B16(16-19) B17(16-19) |
| 3256 | |
| 3257 | const __m256i rhs_mat_0145_13_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_13, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_13 ), (int)(136))); //B10(24-27) B11(24-27) B10(24-27) B11(24-27) B14(24-27) B15(24-27) B14(24-27) B15(24-27) |
| 3258 | const __m256i rhs_mat_2367_13_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_13, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_13 ), (int)(136))); //B12(24-27) B13(24-27) B12(24-27) B13(24-27) B16(24-27) B17(24-27) B16(24-27) B17(24-27) |
| 3259 | |
| 3260 | // Shuffle pattern two - right side input |
| 3261 | const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) |
| 3262 | const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) |
| 3263 | |
| 3264 | const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) |
| 3265 | const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) |
| 3266 | |
| 3267 | const __m256i rhs_mat_0145_02_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_02, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_02 ), (int)(221))); //B00(20-23) B01(20-23) B00(20-23) B01(20-23) B04(20-23) B05(20-23) B04(20-23) B05(20-23) |
| 3268 | const __m256i rhs_mat_2367_02_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_02, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_02 ), (int)(221))); //B02(20-23) B03(20-23) B02(20-23) B03(20-23) B06(20-23) B07(20-23) B06(20-23) B07(20-23) |
| 3269 | |
| 3270 | const __m256i rhs_mat_0145_03_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_03, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_03 ), (int)(221))); //B00(28-31) B01(28-31) B00(28-31) B01(28-31) B04(28-31) B05(28-31) B04(28-31) B05(28-31) |
| 3271 | const __m256i rhs_mat_2367_03_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_03, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_03 ), (int)(221))); //B02(28-31) B03(28-31) B02(28-31) B03(28-31) B06(28-31) B07(28-31) B06(28-31) B07(28-31) |
| 3272 | |
| 3273 | const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) |
| 3274 | const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) |
| 3275 | |
| 3276 | const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) |
| 3277 | const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) |
| 3278 | |
| 3279 | const __m256i rhs_mat_0145_12_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_12, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_12 ), (int)(221))); //B10(20-23) B11(20-23) B10(20-23) B11(20-23) B14(20-23) B15(20-23) B14(20-23) B15(20-23) |
| 3280 | const __m256i rhs_mat_2367_12_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_12, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_12 ), (int)(221))); //B12(20-23) B13(20-23) B12(20-23) B13(20-23) B16(20-23) B17(20-23) B16(20-23) B17(20-23) |
| 3281 | |
| 3282 | const __m256i rhs_mat_0145_13_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_13, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_13 ), (int)(221))); //B10(28-31) B11(28-31) B10(28-31) B11(28-31) B14(28-31) B15(28-31) B14(28-31) B15(28-31) |
| 3283 | const __m256i rhs_mat_2367_13_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_13, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_13 ), (int)(221))); //B12(28-31) B13(28-31) B12(28-31) B13(28-31) B16(28-31) B17(28-31) B16(28-31) B17(28-31) |
| 3284 | |
| 3285 | uint32_t utmp_0[4], utmp_1[4]; |
| 3286 | |
| 3287 | // Scales and Mins of corresponding sub blocks from different Q4_K structures are stored together |
| 3288 | // The below block is for eg to extract first sub block's scales and mins from different Q4_K structures for the sb loop |
| 3289 | memcpy(utmp_0, b_ptr[b].scales + 24 * sb, 12); |
| 3290 | utmp_0[3] = ((utmp_0[2] >> 4) & kmask2) | (((utmp_0[1] >> 6) & kmask3) << 4); |
| 3291 | const uint32_t uaux_0 = utmp_0[1] & kmask1; |
| 3292 | utmp_0[1] = (utmp_0[2] & kmask2) | (((utmp_0[0] >> 6) & kmask3) << 4); |
| 3293 | utmp_0[2] = uaux_0; |
| 3294 | utmp_0[0] &= kmask1; |
| 3295 | |
| 3296 | // The below block is for eg to extract second sub block's scales and mins from different Q4_K structures when sb = 1 |
| 3297 | memcpy(utmp_1, b_ptr[b].scales + 12 + sb * 24, 12); |
| 3298 | utmp_1[3] = ((utmp_1[2] >> 4) & kmask2) | (((utmp_1[1] >> 6) & kmask3) << 4); |
| 3299 | const uint32_t uaux_1 = utmp_1[1] & kmask1; |
| 3300 | utmp_1[1] = (utmp_1[2] & kmask2) | (((utmp_1[0] >> 6) & kmask3) << 4); |
| 3301 | utmp_1[2] = uaux_1; |
| 3302 | utmp_1[0] &= kmask1; |
| 3303 | |
| 3304 | // Scales of first sub block in the sb loop |
| 3305 | const __m128i mins_and_scales_0 = _mm_set_epi32(utmp_0[3], utmp_0[2], utmp_0[1], utmp_0[0]); |
| 3306 | const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_0, mins_and_scales_0)); |
| 3307 | |
| 3308 | // Scales of second sub block in the sb loop |
| 3309 | const __m128i mins_and_scales_1 = _mm_set_epi32(utmp_1[3], utmp_1[2], utmp_1[1], utmp_1[0]); |
| 3310 | const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(mins_and_scales_1, mins_and_scales_1)); |
| 3311 | |
| 3312 | // Mins of first and second sub block of Q4_K block are arranged side by side |
| 3313 | const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_unpacklo_epi8(_mm_shuffle_epi32(mins_and_scales_0, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_0 ), (int)(78))), _mm_shuffle_epi32(mins_and_scales_1, 78)((__m128i)__builtin_ia32_pshufd((__v4si)(__m128i)(mins_and_scales_1 ), (int)(78))))); |
| 3314 | |
| 3315 | const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(68))); |
| 3316 | const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(238))); |
| 3317 | |
| 3318 | const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(68))); |
| 3319 | const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(238))); |
| 3320 | |
| 3321 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 3322 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 3323 | __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 * sb))); |
| 3324 | __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(0))); |
| 3325 | __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(17))); |
| 3326 | __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 256 * sb))); |
| 3327 | __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(0))); |
| 3328 | __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(17))); |
| 3329 | __m256i lhs_mat_0123_02 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 256 * sb))); |
| 3330 | __m256i lhs_mat_01_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_02 ), (__v8si)(__m256i)(lhs_mat_0123_02), (int)(0))); |
| 3331 | __m256i lhs_mat_23_02 = _mm256_permute2f128_si256(lhs_mat_0123_02, lhs_mat_0123_02, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_02 ), (__v8si)(__m256i)(lhs_mat_0123_02), (int)(17))); |
| 3332 | __m256i lhs_mat_0123_03 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 256 * sb))); |
| 3333 | __m256i lhs_mat_01_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_03 ), (__v8si)(__m256i)(lhs_mat_0123_03), (int)(0))); |
| 3334 | __m256i lhs_mat_23_03 = _mm256_permute2f128_si256(lhs_mat_0123_03, lhs_mat_0123_03, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_03 ), (__v8si)(__m256i)(lhs_mat_0123_03), (int)(17))); |
| 3335 | __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 256 * sb))); |
| 3336 | __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(0))); |
| 3337 | __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(17))); |
| 3338 | __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 256 * sb))); |
| 3339 | __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(0))); |
| 3340 | __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(17))); |
| 3341 | __m256i lhs_mat_0123_12 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 256 * sb))); |
| 3342 | __m256i lhs_mat_01_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_12 ), (__v8si)(__m256i)(lhs_mat_0123_12), (int)(0))); |
| 3343 | __m256i lhs_mat_23_12 = _mm256_permute2f128_si256(lhs_mat_0123_12, lhs_mat_0123_12, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_12 ), (__v8si)(__m256i)(lhs_mat_0123_12), (int)(17))); |
| 3344 | __m256i lhs_mat_0123_13 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 256 * sb))); |
| 3345 | __m256i lhs_mat_01_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_13 ), (__v8si)(__m256i)(lhs_mat_0123_13), (int)(0))); |
| 3346 | __m256i lhs_mat_23_13 = _mm256_permute2f128_si256(lhs_mat_0123_13, lhs_mat_0123_13, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_13 ), (__v8si)(__m256i)(lhs_mat_0123_13), (int)(17))); |
| 3347 | |
| 3348 | // Bsums are loaded - four bsums are loaded (for two sub blocks) for the different Q8_K blocks |
| 3349 | __m256i lhs_bsums_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].bsums + 16 * sb))); |
| 3350 | __m256i lhs_bsums_hsum_0123_01 = _mm256_castsi128_si256(_mm_hadd_epi16(_mm256_castsi256_si128(lhs_bsums_0123_01), _mm256_extractf128_si256(lhs_bsums_0123_01, 1)((__m128i)__builtin_ia32_vextractf128_si256((__v8si)(__m256i) (lhs_bsums_0123_01), (int)(1))))); |
| 3351 | lhs_bsums_hsum_0123_01 = _mm256_permute2x128_si256(lhs_bsums_hsum_0123_01, lhs_bsums_hsum_0123_01, 0)((__m256i)__builtin_ia32_permti256((__m256i)(lhs_bsums_hsum_0123_01 ), (__m256i)(lhs_bsums_hsum_0123_01), (int)(0))); |
| 3352 | |
| 3353 | // Shuffle pattern one - left side input |
| 3354 | const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 3355 | const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(160))); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) |
| 3356 | |
| 3357 | const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 3358 | const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(160))); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) |
| 3359 | |
| 3360 | const __m256i lhs_mat_01_02_sp1 = _mm256_shuffle_epi32(lhs_mat_01_02, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_02 ), (int)(160))); //A00(16-19) A00(16-19) A01(16-19) A01(16-19) A00(16-19) A00(16-19) A01(16-19) A01(16-19) |
| 3361 | const __m256i lhs_mat_23_02_sp1 = _mm256_shuffle_epi32(lhs_mat_23_02, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_02 ), (int)(160))); //A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) A02(16-19) A03(16-19) |
| 3362 | |
| 3363 | const __m256i lhs_mat_01_03_sp1 = _mm256_shuffle_epi32(lhs_mat_01_03, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_03 ), (int)(160))); //A00(24-27) A00(24-27) A01(24-27) A01(24-27) A00(24-27) A00(24-27) A01(24-27) A01(24-27) |
| 3364 | const __m256i lhs_mat_23_03_sp1 = _mm256_shuffle_epi32(lhs_mat_23_03, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_03 ), (int)(160))); //A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) A02(24-27) A03(24-27) |
| 3365 | |
| 3366 | const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 3367 | const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(160))); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) |
| 3368 | |
| 3369 | const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 3370 | const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(160))); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) |
| 3371 | |
| 3372 | const __m256i lhs_mat_01_12_sp1 = _mm256_shuffle_epi32(lhs_mat_01_12, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_12 ), (int)(160))); //A10(16-19) A10(16-19) A11(16-19) A11(16-19) A10(16-19) A10(16-19) A11(16-19) A11(16-19) |
| 3373 | const __m256i lhs_mat_23_12_sp1 = _mm256_shuffle_epi32(lhs_mat_23_12, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_12 ), (int)(160))); //A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) A12(16-19) A13(16-19) |
| 3374 | |
| 3375 | const __m256i lhs_mat_01_13_sp1 = _mm256_shuffle_epi32(lhs_mat_01_13, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_13 ), (int)(160))); //A10(24-27) A10(24-27) A11(24-27) A11(24-27) A10(24-27) A10(24-27) A11(24-27) A11(24-27) |
| 3376 | const __m256i lhs_mat_23_13_sp1 = _mm256_shuffle_epi32(lhs_mat_23_13, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_13 ), (int)(160))); //A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) A12(24-27) A13(24-27) |
| 3377 | |
| 3378 | // Shuffle pattern two- left side input |
| 3379 | const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 3380 | const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(245))); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) |
| 3381 | |
| 3382 | const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 3383 | const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(245))); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) |
| 3384 | |
| 3385 | const __m256i lhs_mat_01_02_sp2 = _mm256_shuffle_epi32(lhs_mat_01_02, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_02 ), (int)(245))); //A00(20-23) A00(20-23) A01(20-23) A01(20-23) A00(20-23) A00(20-23) A01(20-23) A01(20-23) |
| 3386 | const __m256i lhs_mat_23_02_sp2 = _mm256_shuffle_epi32(lhs_mat_23_02, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_02 ), (int)(245))); //A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) A02(20-23) A03(20-23) |
| 3387 | |
| 3388 | const __m256i lhs_mat_01_03_sp2 = _mm256_shuffle_epi32(lhs_mat_01_03, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_03 ), (int)(245))); //A00(28-31) A00(28-31) A01(28-31) A01(28-31) A00(28-31) A00(28-31) A01(28-31) A01(28-31) |
| 3389 | const __m256i lhs_mat_23_03_sp2 = _mm256_shuffle_epi32(lhs_mat_23_03, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_03 ), (int)(245))); //A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) A02(28-31) A03(28-31) |
| 3390 | |
| 3391 | const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 3392 | const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(245))); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) |
| 3393 | |
| 3394 | const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 3395 | const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(245))); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) |
| 3396 | |
| 3397 | const __m256i lhs_mat_01_12_sp2 = _mm256_shuffle_epi32(lhs_mat_01_12, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_12 ), (int)(245))); //A10(20-23) A10(20-23) A11(20-23) A11(20-23) A10(20-23) A10(20-23) A11(20-23) A11(20-23) |
| 3398 | const __m256i lhs_mat_23_12_sp2 = _mm256_shuffle_epi32(lhs_mat_23_12, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_12 ), (int)(245))); //A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) A12(20-23) A13(20-23) |
| 3399 | |
| 3400 | const __m256i lhs_mat_01_13_sp2 = _mm256_shuffle_epi32(lhs_mat_01_13, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_13 ), (int)(245))); //A10(28-31) A10(28-31) A11(28-31) A11(28-31) A10(28-31) A10(28-31) A11(28-31) A11(28-31) |
| 3401 | const __m256i lhs_mat_23_13_sp2 = _mm256_shuffle_epi32(lhs_mat_23_13, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_13 ), (int)(245))); //A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) A12(28-31) A13(28-31) |
| 3402 | |
| 3403 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 3404 | __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1)); |
| 3405 | __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_01_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_01_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1)); |
| 3406 | __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_0145_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1)); |
| 3407 | __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp1, lhs_mat_23_03_sp1), _mm256_maddubs_epi16(rhs_mat_2367_02_sp1, lhs_mat_23_02_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1)); |
| 3408 | __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1)); |
| 3409 | __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_01_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_01_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1)); |
| 3410 | __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_0145_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1)); |
| 3411 | __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp1, lhs_mat_23_13_sp1), _mm256_maddubs_epi16(rhs_mat_2367_12_sp1, lhs_mat_23_12_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1)); |
| 3412 | |
| 3413 | __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2)); |
| 3414 | __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_01_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_01_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2)); |
| 3415 | __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_0145_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2)); |
| 3416 | __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_03_sp2, lhs_mat_23_03_sp2), _mm256_maddubs_epi16(rhs_mat_2367_02_sp2, lhs_mat_23_02_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2)); |
| 3417 | __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2)); |
| 3418 | __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_01_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_01_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2)); |
| 3419 | __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_0145_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2)); |
| 3420 | __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_13_sp2, lhs_mat_23_13_sp2), _mm256_maddubs_epi16(rhs_mat_2367_12_sp2, lhs_mat_23_12_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)), _mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2)); |
| 3421 | |
| 3422 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 3423 | __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 3424 | __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 3425 | __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 3426 | __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 3427 | |
| 3428 | __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 3429 | __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 3430 | __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 3431 | __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 3432 | |
| 3433 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 3434 | iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); |
| 3435 | iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); |
| 3436 | iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); |
| 3437 | iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); |
| 3438 | |
| 3439 | iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); |
| 3440 | iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); |
| 3441 | iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); |
| 3442 | iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); |
| 3443 | |
| 3444 | // Straighten out to make 4 row vectors (4 for each sub block which are accumulated together in the next step) |
| 3445 | __m256i iacc_row_0_0 = _mm256_blend_epi32(iacc_mat_00_0, _mm256_shuffle_epi32(iacc_mat_01_0, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00_0 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01_0), (int)(78)))), (int)(204))); |
| 3446 | __m256i iacc_row_1_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_0, 78), iacc_mat_01_0, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00_0), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_01_0), (int)(204))); |
| 3447 | __m256i iacc_row_2_0 = _mm256_blend_epi32(iacc_mat_10_0, _mm256_shuffle_epi32(iacc_mat_11_0, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10_0 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11_0), (int)(78)))), (int)(204))); |
| 3448 | __m256i iacc_row_3_0 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_0, 78), iacc_mat_11_0, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10_0), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_11_0), (int)(204))); |
| 3449 | __m256i iacc_row_0_1 = _mm256_blend_epi32(iacc_mat_00_1, _mm256_shuffle_epi32(iacc_mat_01_1, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00_1 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01_1), (int)(78)))), (int)(204))); |
| 3450 | __m256i iacc_row_1_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00_1, 78), iacc_mat_01_1, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00_1), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_01_1), (int)(204))); |
| 3451 | __m256i iacc_row_2_1 = _mm256_blend_epi32(iacc_mat_10_1, _mm256_shuffle_epi32(iacc_mat_11_1, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10_1 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11_1), (int)(78)))), (int)(204))); |
| 3452 | __m256i iacc_row_3_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10_1, 78), iacc_mat_11_1, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10_1), ( int)(78)))), (__v8si)(__m256i)(iacc_mat_11_1), (int)(204))); |
| 3453 | |
| 3454 | __m256i iacc_row_0 = _mm256_add_epi32(iacc_row_0_0, iacc_row_0_1); |
| 3455 | __m256i iacc_row_1 = _mm256_add_epi32(iacc_row_1_0, iacc_row_1_1); |
| 3456 | __m256i iacc_row_2 = _mm256_add_epi32(iacc_row_2_0, iacc_row_2_1); |
| 3457 | __m256i iacc_row_3 = _mm256_add_epi32(iacc_row_3_0, iacc_row_3_1); |
| 3458 | |
| 3459 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 3460 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); |
| 3461 | const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); //GGML_F32Cx8_REPEAT_LOAD(a_ptrs[rp][b].d, loadMask); |
| 3462 | |
| 3463 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 3464 | acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 3465 | acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 3466 | acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 3467 | acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 3468 | |
| 3469 | __m256i iacc_row_min_0 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(0))), mins_01); |
| 3470 | __m256i iacc_row_min_1 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(85))), mins_01); |
| 3471 | __m256i iacc_row_min_2 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(170))), mins_01); |
| 3472 | __m256i iacc_row_min_3 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_hsum_0123_01, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_hsum_0123_01 ), (int)(255))), mins_01); |
| 3473 | |
| 3474 | acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_min_rows[0]); |
| 3475 | acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_min_rows[1]); |
| 3476 | acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_min_rows[2]); |
| 3477 | acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_min_rows[3]); |
| 3478 | } |
| 3479 | } |
| 3480 | |
| 3481 | // Store the accumulated values |
| 3482 | for (int i = 0; i < 4; i++) { |
| 3483 | _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 3484 | } |
| 3485 | } |
| 3486 | } |
| 3487 | |
| 3488 | #else |
| 3489 | UNUSED(kmask1)(void)(kmask1); |
| 3490 | UNUSED(kmask2)(void)(kmask2); |
| 3491 | UNUSED(kmask3)(void)(kmask3); |
| 3492 | ggml_gemm_q4_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc); |
| 3493 | #endif |
| 3494 | } |
| 3495 | |
| 3496 | void ggml_gemm_iq4_nl_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 3497 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 3498 | { |
| 3499 | __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_iq4nl)); |
| 3500 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 3501 | |
| 3502 | gemm_q4_b32_8x8_q8_0_lut_avx<block_iq4_nlx8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 3503 | |
| 3504 | return; |
| 3505 | } |
| 3506 | #endif // defined(__AVX2__) || defined(__AVX512F__) |
| 3507 | |
| 3508 | ggml_gemm_iq4_nl_4x4_q8_0(n, s, bs, vx, vy, nr, nc); |
| 3509 | } |
| 3510 | |
| 3511 | void ggml_gemm_mxfp4_8x8_q8_0(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 3512 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 3513 | { |
| 3514 | __m256i signextendlut = _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)kvalues_mxfp4)); |
| 3515 | signextendlut = _mm256_permute2f128_si256(signextendlut, signextendlut, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(signextendlut ), (__v8si)(__m256i)(signextendlut), (int)(0))); |
| 3516 | |
| 3517 | gemm_q4_b32_8x8_q8_0_lut_avx<block_mxfp4x8>(n, s, bs, vx, vy, nr, nc, signextendlut); |
| 3518 | |
| 3519 | return; |
| 3520 | } |
| 3521 | #endif // defined(__AVX2__) || defined(__AVX512F__) |
| 3522 | |
| 3523 | ggml_gemm_mxfp4_8x8_q8_0_generic(n, s, bs, vx, vy, nr, nc); |
| 3524 | } |
| 3525 | |
| 3526 | void ggml_gemm_q2_K_8x8_q8_K(int n, float * GGML_RESTRICT__restrict__ s, size_t bs, const void * GGML_RESTRICT__restrict__ vx, const void * GGML_RESTRICT__restrict__ vy, int nr, int nc) { |
| 3527 | const int qk = QK_K256; |
| 3528 | const int nb = n / qk; |
| 3529 | const int ncols_interleaved = 8; |
| 3530 | const int blocklen = 8; |
| 3531 | |
| 3532 | assert (n % qk == 0)(static_cast <bool> (n % qk == 0) ? void (0) : __assert_fail ("n % qk == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 3533 | assert (nr % 4 == 0)(static_cast <bool> (nr % 4 == 0) ? void (0) : __assert_fail ("nr % 4 == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 3534 | assert (nc % ncols_interleaved == 0)(static_cast <bool> (nc % ncols_interleaved == 0) ? void (0) : __assert_fail ("nc % ncols_interleaved == 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 3535 | |
| 3536 | UNUSED(s)(void)(s); |
| 3537 | UNUSED(bs)(void)(bs); |
| 3538 | UNUSED(vx)(void)(vx); |
| 3539 | UNUSED(vy)(void)(vy); |
| 3540 | UNUSED(nr)(void)(nr); |
| 3541 | UNUSED(nc)(void)(nc); |
| 3542 | UNUSED(nb)(void)(nb); |
| 3543 | UNUSED(ncols_interleaved)(void)(ncols_interleaved); |
| 3544 | UNUSED(blocklen)(void)(blocklen); |
| 3545 | |
| 3546 | #if defined(__AVX2__1) || defined(__AVX512F__) |
| 3547 | const block_q2_Kx8 * b_ptr_start = (const block_q2_Kx8 * ) vx; |
| 3548 | const block_q8_Kx4 * a_ptr_start = (const block_q8_Kx4 * ) vy; |
| 3549 | int64_t b_nb = n / QK_K256; |
| 3550 | int64_t y = 0; |
| 3551 | |
| 3552 | // Permute mask used for easier vector processing at later stages |
| 3553 | __m256i requiredOrder = _mm256_set_epi32(3, 2, 1, 0, 7, 6, 5, 4); |
| 3554 | int64_t xstart = 0; |
| 3555 | int anr = nr - nr % 16; // Used to align nr with boundary of 16 |
| 3556 | |
| 3557 | // Mask to convert 2 bit and 4 bit values into a bytes |
| 3558 | const __m256i m3b = _mm256_set1_epi8(3); |
| 3559 | const __m128i m4b_sse = _mm_set1_epi8(0xF); |
| 3560 | |
| 3561 | //Mask to get appropriate scales |
| 3562 | __m128i scalesmask1_sse = _mm_set_epi8(14,14,12,12,10,10,8,8,6,6,4,4,2,2,0,0); |
| 3563 | __m128i scalesmask2_sse = _mm_set_epi8(15,15,13,13,11,11,9,9,7,7,5,5,3,3,1,1); |
| 3564 | |
| 3565 | __m256i scalesmask1 = _mm256_castsi128_si256(scalesmask1_sse); |
| 3566 | scalesmask1 = _mm256_permute2f128_si256(scalesmask1, scalesmask1, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(scalesmask1 ), (__v8si)(__m256i)(scalesmask1), (int)(0))); |
Value stored to 'scalesmask1' is never read | |
| 3567 | __m256i scalesmask2 = _mm256_castsi128_si256(scalesmask2_sse); |
| 3568 | scalesmask2 = _mm256_permute2f128_si256(scalesmask2, scalesmask2, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(scalesmask2 ), (__v8si)(__m256i)(scalesmask2), (int)(0))); |
| 3569 | |
| 3570 | #if defined(__AVX512BW__) && defined(__AVX512DQ__) |
| 3571 | |
| 3572 | int anc = nc - nc % 16; // Used to align nc with boundary of 16 |
| 3573 | |
| 3574 | // Mask to mask out nibbles from packed bytes |
| 3575 | const __m256i m4b = _mm256_set1_epi8(0x0F); |
| 3576 | // Mask to mask out nibbles from packed bytes expanded to 512 bit length |
| 3577 | const __m512i m3bexpanded = _mm512_set1_epi8(3); |
| 3578 | //Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation |
| 3579 | for (; y < anr / 4; y += 4) { |
| 3580 | |
| 3581 | const block_q8_Kx4 * a_ptrs[4]; |
| 3582 | |
| 3583 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 3584 | for (int i = 0; i < 3; ++i) { |
| 3585 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 3586 | } |
| 3587 | |
| 3588 | // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation |
| 3589 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 3590 | |
| 3591 | const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 3592 | const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 3593 | |
| 3594 | // Master FP accumulators |
| 3595 | __m512 acc_rows[16]; |
| 3596 | for (int i = 0; i < 16; i++) { |
| 3597 | acc_rows[i] = _mm512_setzero_ps(); |
| 3598 | } |
| 3599 | |
| 3600 | __m512 acc_min_rows[16]; |
| 3601 | for (int i = 0; i < 16; i++) { |
| 3602 | acc_min_rows[i] = _mm512_setzero_ps(); |
| 3603 | } |
| 3604 | // For super block |
| 3605 | for (int64_t b = 0; b < nb; b++) { |
| 3606 | // Delta values - Load the sixteen scale values from two block_q2_kx8 structures |
| 3607 | const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 3608 | |
| 3609 | // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures |
| 3610 | const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin); |
| 3611 | |
| 3612 | // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration |
| 3613 | for (int sb = 0; sb < QK_K256 / 128; sb++) { |
| 3614 | |
| 3615 | // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 3616 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256)); |
| 3617 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256)); |
| 3618 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256)); |
| 3619 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256)); |
| 3620 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256)); |
| 3621 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256)); |
| 3622 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256)); |
| 3623 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256)); |
| 3624 | |
| 3625 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256)); |
| 3626 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256)); |
| 3627 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256)); |
| 3628 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256)); |
| 3629 | const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256)); |
| 3630 | const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256)); |
| 3631 | const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256)); |
| 3632 | const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256)); |
| 3633 | |
| 3634 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 3635 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 3636 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 3637 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 3638 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 3639 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 3640 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 3641 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 3642 | |
| 3643 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 3644 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 3645 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 3646 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 3647 | const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2 , requiredOrder)), (int)(240))); |
| 3648 | const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_2 ), (int)(240))); |
| 3649 | const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3 , requiredOrder)), (int)(240))); |
| 3650 | const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_3 ), (int)(240))); |
| 3651 | |
| 3652 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 3653 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 3654 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 3655 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 3656 | |
| 3657 | const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_2)), (__v8si)(__m256i)(rhs_raw_mat_89CD_2), (int)(1))); |
| 3658 | const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_2)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_2), (int)(1))); |
| 3659 | const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_3)), (__v8si)(__m256i)(rhs_raw_mat_89CD_3), (int)(1))); |
| 3660 | const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_3)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_3), (int)(1))); |
| 3661 | |
| 3662 | //2-bit -> 8-bit |
| 3663 | const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7) |
| 3664 | const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7) |
| 3665 | const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15) |
| 3666 | const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15) |
| 3667 | const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7) |
| 3668 | const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7) |
| 3669 | const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15) |
| 3670 | const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15) |
| 3671 | |
| 3672 | const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7) |
| 3673 | const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7) |
| 3674 | |
| 3675 | const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15) |
| 3676 | const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15) |
| 3677 | |
| 3678 | const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7) |
| 3679 | const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7) |
| 3680 | |
| 3681 | const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15) |
| 3682 | const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15) |
| 3683 | |
| 3684 | const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7) |
| 3685 | const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7) |
| 3686 | |
| 3687 | const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15) |
| 3688 | const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15) |
| 3689 | |
| 3690 | const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7) |
| 3691 | const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7) |
| 3692 | |
| 3693 | const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15) |
| 3694 | const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15) |
| 3695 | |
| 3696 | const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7) |
| 3697 | const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7) |
| 3698 | |
| 3699 | const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15) |
| 3700 | const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15) |
| 3701 | |
| 3702 | const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7) |
| 3703 | const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7) |
| 3704 | |
| 3705 | const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15) |
| 3706 | const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15) |
| 3707 | |
| 3708 | const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3) |
| 3709 | const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3) |
| 3710 | |
| 3711 | const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 3712 | const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11) |
| 3713 | |
| 3714 | const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3) |
| 3715 | const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3) |
| 3716 | |
| 3717 | const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11) |
| 3718 | const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11) |
| 3719 | |
| 3720 | const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_20 ), (int)((_MM_PERM_ENUM)136))); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3) |
| 3721 | const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_20 ), (int)((_MM_PERM_ENUM)136))); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3) |
| 3722 | |
| 3723 | const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_21 ), (int)((_MM_PERM_ENUM)136))); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11) |
| 3724 | const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_21 ), (int)((_MM_PERM_ENUM)136))); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11) |
| 3725 | |
| 3726 | const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_30 ), (int)((_MM_PERM_ENUM)136))); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3) |
| 3727 | const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_30 ), (int)((_MM_PERM_ENUM)136))); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3) |
| 3728 | |
| 3729 | const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_31 ), (int)((_MM_PERM_ENUM)136))); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11) |
| 3730 | const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_31 ), (int)((_MM_PERM_ENUM)136))); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11) |
| 3731 | |
| 3732 | const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_40 ), (int)((_MM_PERM_ENUM)136))); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3) |
| 3733 | const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_40 ), (int)((_MM_PERM_ENUM)136))); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3) |
| 3734 | |
| 3735 | const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_41 ), (int)((_MM_PERM_ENUM)136))); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11) |
| 3736 | const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_41 ), (int)((_MM_PERM_ENUM)136))); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11) |
| 3737 | |
| 3738 | const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_50 ), (int)((_MM_PERM_ENUM)136))); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3) |
| 3739 | const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_50 ), (int)((_MM_PERM_ENUM)136))); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3) |
| 3740 | |
| 3741 | const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_51 ), (int)((_MM_PERM_ENUM)136))); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11) |
| 3742 | const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_51 ), (int)((_MM_PERM_ENUM)136))); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11) |
| 3743 | |
| 3744 | const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_60 ), (int)((_MM_PERM_ENUM)136))); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3) |
| 3745 | const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_60 ), (int)((_MM_PERM_ENUM)136))); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3) |
| 3746 | |
| 3747 | const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_61 ), (int)((_MM_PERM_ENUM)136))); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11) |
| 3748 | const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_61 ), (int)((_MM_PERM_ENUM)136))); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11) |
| 3749 | |
| 3750 | const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_70 ), (int)((_MM_PERM_ENUM)136))); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3) |
| 3751 | const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_70 ), (int)((_MM_PERM_ENUM)136))); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3) |
| 3752 | |
| 3753 | const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_71 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 3754 | const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_71 ), (int)((_MM_PERM_ENUM)136))); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11) |
| 3755 | |
| 3756 | const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7) |
| 3757 | const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7) |
| 3758 | |
| 3759 | const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15) |
| 3760 | const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15) |
| 3761 | |
| 3762 | const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7) |
| 3763 | const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7) |
| 3764 | |
| 3765 | const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15) |
| 3766 | const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15) |
| 3767 | |
| 3768 | const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_20 ), (int)((_MM_PERM_ENUM)221))); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7) |
| 3769 | const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_20 ), (int)((_MM_PERM_ENUM)221))); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7) |
| 3770 | |
| 3771 | const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_21 ), (int)((_MM_PERM_ENUM)221))); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15) |
| 3772 | const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_21 ), (int)((_MM_PERM_ENUM)221))); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15) |
| 3773 | |
| 3774 | const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_30 ), (int)((_MM_PERM_ENUM)221))); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7) |
| 3775 | const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_30 ), (int)((_MM_PERM_ENUM)221))); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7) |
| 3776 | |
| 3777 | const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_31 ), (int)((_MM_PERM_ENUM)221))); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15) |
| 3778 | const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_31 ), (int)((_MM_PERM_ENUM)221))); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15) |
| 3779 | |
| 3780 | const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_40 ), (int)((_MM_PERM_ENUM)221))); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7) |
| 3781 | const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_40 ), (int)((_MM_PERM_ENUM)221))); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7) |
| 3782 | |
| 3783 | const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_41 ), (int)((_MM_PERM_ENUM)221))); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15) |
| 3784 | const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_41 ), (int)((_MM_PERM_ENUM)221))); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15) |
| 3785 | |
| 3786 | const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_50 ), (int)((_MM_PERM_ENUM)221))); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7) |
| 3787 | const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_50 ), (int)((_MM_PERM_ENUM)221))); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7) |
| 3788 | |
| 3789 | const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_51 ), (int)((_MM_PERM_ENUM)221))); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15) |
| 3790 | const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_51 ), (int)((_MM_PERM_ENUM)221))); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15) |
| 3791 | |
| 3792 | const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_60 ), (int)((_MM_PERM_ENUM)221))); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7) |
| 3793 | const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_60 ), (int)((_MM_PERM_ENUM)221))); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7) |
| 3794 | |
| 3795 | const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_61 ), (int)((_MM_PERM_ENUM)221))); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15) |
| 3796 | const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_61 ), (int)((_MM_PERM_ENUM)221))); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15) |
| 3797 | |
| 3798 | const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_70 ), (int)((_MM_PERM_ENUM)221))); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7) |
| 3799 | const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_70 ), (int)((_MM_PERM_ENUM)221))); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7) |
| 3800 | |
| 3801 | const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_71 ), (int)((_MM_PERM_ENUM)221))); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15) |
| 3802 | const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_71 ), (int)((_MM_PERM_ENUM)221))); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15) |
| 3803 | |
| 3804 | //notation:superblock subblock |
| 3805 | //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71 |
| 3806 | |
| 3807 | const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64)); |
| 3808 | const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64)); |
| 3809 | const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64)); |
| 3810 | const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64)); |
| 3811 | |
| 3812 | const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64)); |
| 3813 | const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64)); |
| 3814 | const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64)); |
| 3815 | const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64)); |
| 3816 | |
| 3817 | // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop |
| 3818 | const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_01_0)), (__v4si)(__m128i )(mins_and_scales_01_1), (int)(1))); |
| 3819 | const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_23_0)), (__v4si)(__m128i )(mins_and_scales_23_1), (int)(1))); |
| 3820 | const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_45_0)), (__v4si)(__m128i )(mins_and_scales_45_1), (int)(1))); |
| 3821 | const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_67_0)), (__v4si)(__m128i )(mins_and_scales_67_1), (int)(1))); |
| 3822 | |
| 3823 | // Extract scales which is lower half from mins_and_scales |
| 3824 | const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b); |
| 3825 | const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b); |
| 3826 | const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b); |
| 3827 | const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b); |
| 3828 | |
| 3829 | // Extract mins which is upper half from mins_and_scales |
| 3830 | const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b)); |
| 3831 | const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b)); |
| 3832 | const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b)); |
| 3833 | const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b)); |
| 3834 | |
| 3835 | const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask1)); |
| 3836 | const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01,scalesmask2)); |
| 3837 | const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask1)); |
| 3838 | const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23,scalesmask2)); |
| 3839 | const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask1)); |
| 3840 | const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45,scalesmask2)); |
| 3841 | const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask1)); |
| 3842 | const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67,scalesmask2)); |
| 3843 | |
| 3844 | const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)68))); |
| 3845 | const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)238))); |
| 3846 | |
| 3847 | const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)68))); |
| 3848 | const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)238))); |
| 3849 | |
| 3850 | const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_2 ), (int)((_MM_PERM_ENUM)68))); |
| 3851 | const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_2 ), (int)((_MM_PERM_ENUM)238))); |
| 3852 | |
| 3853 | const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_3 ), (int)((_MM_PERM_ENUM)68))); |
| 3854 | const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_3 ), (int)((_MM_PERM_ENUM)238))); |
| 3855 | |
| 3856 | const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_4 ), (int)((_MM_PERM_ENUM)68))); |
| 3857 | const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_4 ), (int)((_MM_PERM_ENUM)238))); |
| 3858 | |
| 3859 | const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_5 ), (int)((_MM_PERM_ENUM)68))); |
| 3860 | const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_5 ), (int)((_MM_PERM_ENUM)238))); |
| 3861 | |
| 3862 | const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_6 ), (int)((_MM_PERM_ENUM)68))); |
| 3863 | const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_6 ), (int)((_MM_PERM_ENUM)238))); |
| 3864 | |
| 3865 | const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_7 ), (int)((_MM_PERM_ENUM)68))); |
| 3866 | const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_7 ), (int)((_MM_PERM_ENUM)238))); |
| 3867 | |
| 3868 | |
| 3869 | for (int rp = 0; rp < 4; rp++) { |
| 3870 | |
| 3871 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 3872 | // Loaded as set of 128 bit vectors and repeated and stored into a 256 bit vector before again repeating into 512 bit vector |
| 3873 | __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb))); |
| 3874 | __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(0))); |
| 3875 | __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(17))); |
| 3876 | __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb))); |
| 3877 | __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(0))); |
| 3878 | __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(17))); |
| 3879 | __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb))); |
| 3880 | __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(0))); |
| 3881 | __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(17))); |
| 3882 | __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb))); |
| 3883 | __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(0))); |
| 3884 | __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(17))); |
| 3885 | __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb))); |
| 3886 | __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_20 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_20), (int)(0))); |
| 3887 | __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_20 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_20), (int)(17))); |
| 3888 | __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb))); |
| 3889 | __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_21 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_21), (int)(0))); |
| 3890 | __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_21 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_21), (int)(17))); |
| 3891 | __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb))); |
| 3892 | __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_30 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_30), (int)(0))); |
| 3893 | __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_30 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_30), (int)(17))); |
| 3894 | __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb))); |
| 3895 | __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_31 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_31), (int)(0))); |
| 3896 | __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_31 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_31), (int)(17))); |
| 3897 | |
| 3898 | __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb))); |
| 3899 | __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_40 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_40), (int)(0))); |
| 3900 | __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_40 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_40), (int)(17))); |
| 3901 | __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb))); |
| 3902 | __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_41 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_41), (int)(0))); |
| 3903 | __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_41 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_41), (int)(17))); |
| 3904 | __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb))); |
| 3905 | __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_50 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_50), (int)(0))); |
| 3906 | __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_50 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_50), (int)(17))); |
| 3907 | __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb))); |
| 3908 | __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_51 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_51), (int)(0))); |
| 3909 | __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_51 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_51), (int)(17))); |
| 3910 | __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb))); |
| 3911 | __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_60 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_60), (int)(0))); |
| 3912 | __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_60 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_60), (int)(17))); |
| 3913 | __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb))); |
| 3914 | __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_61 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_61), (int)(0))); |
| 3915 | __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_61 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_61), (int)(17))); |
| 3916 | __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb))); |
| 3917 | __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_70 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_70), (int)(0))); |
| 3918 | __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_70 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_70), (int)(17))); |
| 3919 | __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb))); |
| 3920 | __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_71 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_71), (int)(0))); |
| 3921 | __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_71 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_71), (int)(17))); |
| 3922 | |
| 3923 | |
| 3924 | __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_00)), (__v8si)(__m256i)(lhs_mat_ymm_01_00), ( int)(1))); |
| 3925 | __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_00)), (__v8si)(__m256i)(lhs_mat_ymm_23_00), ( int)(1))); |
| 3926 | __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_01)), (__v8si)(__m256i)(lhs_mat_ymm_01_01), ( int)(1))); |
| 3927 | __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_01)), (__v8si)(__m256i)(lhs_mat_ymm_23_01), ( int)(1))); |
| 3928 | |
| 3929 | __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_10)), (__v8si)(__m256i)(lhs_mat_ymm_01_10), ( int)(1))); |
| 3930 | __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_10)), (__v8si)(__m256i)(lhs_mat_ymm_23_10), ( int)(1))); |
| 3931 | __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_11)), (__v8si)(__m256i)(lhs_mat_ymm_01_11), ( int)(1))); |
| 3932 | __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_11)), (__v8si)(__m256i)(lhs_mat_ymm_23_11), ( int)(1))); |
| 3933 | |
| 3934 | __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_20)), (__v8si)(__m256i)(lhs_mat_ymm_01_20), ( int)(1))); |
| 3935 | __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_20)), (__v8si)(__m256i)(lhs_mat_ymm_23_20), ( int)(1))); |
| 3936 | __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_21)), (__v8si)(__m256i)(lhs_mat_ymm_01_21), ( int)(1))); |
| 3937 | __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_21)), (__v8si)(__m256i)(lhs_mat_ymm_23_21), ( int)(1))); |
| 3938 | |
| 3939 | __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_30)), (__v8si)(__m256i)(lhs_mat_ymm_01_30), ( int)(1))); |
| 3940 | __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_30)), (__v8si)(__m256i)(lhs_mat_ymm_23_30), ( int)(1))); |
| 3941 | __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_31)), (__v8si)(__m256i)(lhs_mat_ymm_01_31), ( int)(1))); |
| 3942 | __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_31)), (__v8si)(__m256i)(lhs_mat_ymm_23_31), ( int)(1))); |
| 3943 | |
| 3944 | __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_40)), (__v8si)(__m256i)(lhs_mat_ymm_01_40), ( int)(1))); |
| 3945 | __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_40)), (__v8si)(__m256i)(lhs_mat_ymm_23_40), ( int)(1))); |
| 3946 | __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_41)), (__v8si)(__m256i)(lhs_mat_ymm_01_41), ( int)(1))); |
| 3947 | __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_41)), (__v8si)(__m256i)(lhs_mat_ymm_23_41), ( int)(1))); |
| 3948 | |
| 3949 | __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_50)), (__v8si)(__m256i)(lhs_mat_ymm_01_50), ( int)(1))); |
| 3950 | __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_50)), (__v8si)(__m256i)(lhs_mat_ymm_23_50), ( int)(1))); |
| 3951 | __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_51)), (__v8si)(__m256i)(lhs_mat_ymm_01_51), ( int)(1))); |
| 3952 | __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_51)), (__v8si)(__m256i)(lhs_mat_ymm_23_51), ( int)(1))); |
| 3953 | |
| 3954 | __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_60)), (__v8si)(__m256i)(lhs_mat_ymm_01_60), ( int)(1))); |
| 3955 | __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_60)), (__v8si)(__m256i)(lhs_mat_ymm_23_60), ( int)(1))); |
| 3956 | __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_61)), (__v8si)(__m256i)(lhs_mat_ymm_01_61), ( int)(1))); |
| 3957 | __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_61)), (__v8si)(__m256i)(lhs_mat_ymm_23_61), ( int)(1))); |
| 3958 | |
| 3959 | __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_70)), (__v8si)(__m256i)(lhs_mat_ymm_01_70), ( int)(1))); |
| 3960 | __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_70)), (__v8si)(__m256i)(lhs_mat_ymm_23_70), ( int)(1))); |
| 3961 | __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_71)), (__v8si)(__m256i)(lhs_mat_ymm_01_71), ( int)(1))); |
| 3962 | __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_71)), (__v8si)(__m256i)(lhs_mat_ymm_23_71), ( int)(1))); |
| 3963 | |
| 3964 | // Bsums are loaded for the different Q8_K blocks |
| 3965 | __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb))); |
| 3966 | __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb)); |
| 3967 | __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb))); |
| 3968 | __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb)); |
| 3969 | |
| 3970 | __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_0123)), (__v2di)(__m128i)(lhs_raw_bsums_01_0123 ), (int)(1))); |
| 3971 | __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_01_0123)), (__v8si)(__m256i)(lhs_bsums_ymm_01_0123 ), (int)(1))); |
| 3972 | __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_0123)), (__v2di)(__m128i)(lhs_raw_bsums_23_0123 ), (int)(1))); |
| 3973 | __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_23_0123)), (__v8si)(__m256i)(lhs_bsums_ymm_23_0123 ), (int)(1))); __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_4567)), (__v2di)(__m128i)(lhs_raw_bsums_01_4567 ), (int)(1))); |
| 3974 | __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_01_4567)), (__v8si)(__m256i)(lhs_bsums_ymm_01_4567 ), (int)(1))); |
| 3975 | __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_4567)), (__v2di)(__m128i)(lhs_raw_bsums_23_4567 ), (int)(1))); |
| 3976 | __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_23_4567)), (__v8si)(__m256i)(lhs_bsums_ymm_23_4567 ), (int)(1))); |
| 3977 | |
| 3978 | // Shuffle pattern one - left side input |
| 3979 | const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 3980 | const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)160))); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) |
| 3981 | |
| 3982 | const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 3983 | const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)160))); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) |
| 3984 | |
| 3985 | const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 3986 | const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)160))); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) |
| 3987 | |
| 3988 | const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 3989 | const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)160))); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) |
| 3990 | |
| 3991 | const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_20 ), (int)((_MM_PERM_ENUM)160))); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) |
| 3992 | const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_20 ), (int)((_MM_PERM_ENUM)160))); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) |
| 3993 | |
| 3994 | const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_21 ), (int)((_MM_PERM_ENUM)160))); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) |
| 3995 | const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_21 ), (int)((_MM_PERM_ENUM)160))); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) |
| 3996 | |
| 3997 | const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_30 ), (int)((_MM_PERM_ENUM)160))); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) |
| 3998 | const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_30 ), (int)((_MM_PERM_ENUM)160))); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) |
| 3999 | |
| 4000 | const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_31 ), (int)((_MM_PERM_ENUM)160))); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) |
| 4001 | const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_31 ), (int)((_MM_PERM_ENUM)160))); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) |
| 4002 | |
| 4003 | const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_40 ), (int)((_MM_PERM_ENUM)160))); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) |
| 4004 | const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_40 ), (int)((_MM_PERM_ENUM)160))); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) |
| 4005 | |
| 4006 | const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_41 ), (int)((_MM_PERM_ENUM)160))); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) |
| 4007 | const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_41 ), (int)((_MM_PERM_ENUM)160))); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) |
| 4008 | |
| 4009 | const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_50 ), (int)((_MM_PERM_ENUM)160))); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) |
| 4010 | const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_50 ), (int)((_MM_PERM_ENUM)160))); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) |
| 4011 | |
| 4012 | const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_51 ), (int)((_MM_PERM_ENUM)160))); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) |
| 4013 | const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_51 ), (int)((_MM_PERM_ENUM)160))); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) |
| 4014 | |
| 4015 | const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_60 ), (int)((_MM_PERM_ENUM)160))); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) |
| 4016 | const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_60 ), (int)((_MM_PERM_ENUM)160))); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) |
| 4017 | |
| 4018 | const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_61 ), (int)((_MM_PERM_ENUM)160))); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) |
| 4019 | const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_61 ), (int)((_MM_PERM_ENUM)160))); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) |
| 4020 | |
| 4021 | const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_70 ), (int)((_MM_PERM_ENUM)160))); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) |
| 4022 | const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_70 ), (int)((_MM_PERM_ENUM)160))); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) |
| 4023 | |
| 4024 | const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_71 ), (int)((_MM_PERM_ENUM)160))); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) |
| 4025 | const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_71 ), (int)((_MM_PERM_ENUM)160))); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) |
| 4026 | |
| 4027 | const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 4028 | const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)245))); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) |
| 4029 | |
| 4030 | const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 4031 | const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)245))); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) |
| 4032 | |
| 4033 | const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 4034 | const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)245))); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) |
| 4035 | |
| 4036 | const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 4037 | const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)245))); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) |
| 4038 | |
| 4039 | const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_20 ), (int)((_MM_PERM_ENUM)245))); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) |
| 4040 | const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_20 ), (int)((_MM_PERM_ENUM)245))); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) |
| 4041 | |
| 4042 | const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_21 ), (int)((_MM_PERM_ENUM)245))); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) |
| 4043 | const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_21 ), (int)((_MM_PERM_ENUM)245))); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) |
| 4044 | |
| 4045 | const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_30 ), (int)((_MM_PERM_ENUM)245))); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) |
| 4046 | const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_30 ), (int)((_MM_PERM_ENUM)245))); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) |
| 4047 | |
| 4048 | const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_31 ), (int)((_MM_PERM_ENUM)245))); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) |
| 4049 | const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_31 ), (int)((_MM_PERM_ENUM)245))); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) |
| 4050 | |
| 4051 | const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_40 ), (int)((_MM_PERM_ENUM)245))); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) |
| 4052 | const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_40 ), (int)((_MM_PERM_ENUM)245))); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) |
| 4053 | |
| 4054 | const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_41 ), (int)((_MM_PERM_ENUM)245))); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) |
| 4055 | const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_41 ), (int)((_MM_PERM_ENUM)245))); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) |
| 4056 | |
| 4057 | const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_50 ), (int)((_MM_PERM_ENUM)245))); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) |
| 4058 | const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_50 ), (int)((_MM_PERM_ENUM)245))); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) |
| 4059 | |
| 4060 | const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_51 ), (int)((_MM_PERM_ENUM)245))); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) |
| 4061 | const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_51 ), (int)((_MM_PERM_ENUM)245))); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) |
| 4062 | |
| 4063 | const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_60 ), (int)((_MM_PERM_ENUM)245))); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) |
| 4064 | const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_60 ), (int)((_MM_PERM_ENUM)245))); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) |
| 4065 | |
| 4066 | const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_61 ), (int)((_MM_PERM_ENUM)245))); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) |
| 4067 | const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_61 ), (int)((_MM_PERM_ENUM)245))); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) |
| 4068 | |
| 4069 | const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_70 ), (int)((_MM_PERM_ENUM)245))); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) |
| 4070 | const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_70 ), (int)((_MM_PERM_ENUM)245))); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) |
| 4071 | |
| 4072 | const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_71 ), (int)((_MM_PERM_ENUM)245))); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) |
| 4073 | const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_71 ), (int)((_MM_PERM_ENUM)245))); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) |
| 4074 | |
| 4075 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 4076 | __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)); |
| 4077 | __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)); |
| 4078 | |
| 4079 | __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)); |
| 4080 | __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)); |
| 4081 | |
| 4082 | __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)); |
| 4083 | __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)); |
| 4084 | |
| 4085 | __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)); |
| 4086 | __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)); |
| 4087 | |
| 4088 | __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1)); |
| 4089 | __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1)); |
| 4090 | |
| 4091 | __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1)); |
| 4092 | __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1)); |
| 4093 | |
| 4094 | __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1)); |
| 4095 | __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1)); |
| 4096 | |
| 4097 | __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1)); |
| 4098 | __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1)); |
| 4099 | |
| 4100 | __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1)); |
| 4101 | __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1)); |
| 4102 | |
| 4103 | __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1)); |
| 4104 | __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1)); |
| 4105 | |
| 4106 | __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1)); |
| 4107 | __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1)); |
| 4108 | |
| 4109 | __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1)); |
| 4110 | __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1)); |
| 4111 | |
| 4112 | __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1)); |
| 4113 | __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1)); |
| 4114 | |
| 4115 | __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1)); |
| 4116 | __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1)); |
| 4117 | |
| 4118 | __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1)); |
| 4119 | __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1)); |
| 4120 | |
| 4121 | __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1)); |
| 4122 | __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1)); |
| 4123 | |
| 4124 | |
| 4125 | __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)); |
| 4126 | __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)); |
| 4127 | |
| 4128 | __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)); |
| 4129 | __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)); |
| 4130 | |
| 4131 | __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)); |
| 4132 | __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)); |
| 4133 | |
| 4134 | __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)); |
| 4135 | __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)); |
| 4136 | |
| 4137 | __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2)); |
| 4138 | __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2)); |
| 4139 | |
| 4140 | __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2)); |
| 4141 | __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2)); |
| 4142 | |
| 4143 | __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2)); |
| 4144 | __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2)); |
| 4145 | |
| 4146 | __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2)); |
| 4147 | __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2)); |
| 4148 | |
| 4149 | __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2)); |
| 4150 | __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2)); |
| 4151 | |
| 4152 | __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2)); |
| 4153 | __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2)); |
| 4154 | |
| 4155 | __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2)); |
| 4156 | __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2)); |
| 4157 | |
| 4158 | __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2)); |
| 4159 | __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2)); |
| 4160 | |
| 4161 | __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2)); |
| 4162 | __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2)); |
| 4163 | |
| 4164 | __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2)); |
| 4165 | __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2)); |
| 4166 | |
| 4167 | __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2)); |
| 4168 | __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2)); |
| 4169 | |
| 4170 | __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2)); |
| 4171 | __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2)); |
| 4172 | |
| 4173 | // Combine results from both shuffle patterns for each output block |
| 4174 | __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 4175 | __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 4176 | __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 4177 | __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 4178 | |
| 4179 | __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 4180 | __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 4181 | __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 4182 | __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 4183 | |
| 4184 | __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2); |
| 4185 | __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2); |
| 4186 | __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2); |
| 4187 | __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2); |
| 4188 | |
| 4189 | __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2); |
| 4190 | __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2); |
| 4191 | __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2); |
| 4192 | __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2); |
| 4193 | |
| 4194 | __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2); |
| 4195 | __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2); |
| 4196 | __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2); |
| 4197 | __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2); |
| 4198 | |
| 4199 | __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2); |
| 4200 | __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2); |
| 4201 | __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2); |
| 4202 | __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2); |
| 4203 | |
| 4204 | __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2); |
| 4205 | __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2); |
| 4206 | __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2); |
| 4207 | __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2); |
| 4208 | |
| 4209 | __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2); |
| 4210 | __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2); |
| 4211 | __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2); |
| 4212 | __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2); |
| 4213 | |
| 4214 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 4215 | iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0); |
| 4216 | iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0); |
| 4217 | iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0); |
| 4218 | iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0); |
| 4219 | |
| 4220 | iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1); |
| 4221 | iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1); |
| 4222 | iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1); |
| 4223 | iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1); |
| 4224 | |
| 4225 | iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2); |
| 4226 | iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2); |
| 4227 | iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2); |
| 4228 | iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2); |
| 4229 | |
| 4230 | iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3); |
| 4231 | iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3); |
| 4232 | iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3); |
| 4233 | iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3); |
| 4234 | |
| 4235 | iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4); |
| 4236 | iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4); |
| 4237 | iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4); |
| 4238 | iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4); |
| 4239 | |
| 4240 | iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5); |
| 4241 | iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5); |
| 4242 | iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5); |
| 4243 | iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5); |
| 4244 | |
| 4245 | iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6); |
| 4246 | iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6); |
| 4247 | iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6); |
| 4248 | iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6); |
| 4249 | |
| 4250 | iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7); |
| 4251 | iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7); |
| 4252 | iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7); |
| 4253 | iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7); |
| 4254 | |
| 4255 | __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7))); |
| 4256 | __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7))); |
| 4257 | __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7))); |
| 4258 | __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7))); |
| 4259 | |
| 4260 | // Straighten out to make 4 row vectors |
| 4261 | __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01 ), (int)((_MM_PERM_ENUM)78)))); |
| 4262 | __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01); |
| 4263 | __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11 ), (int)((_MM_PERM_ENUM)78)))); |
| 4264 | __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11); |
| 4265 | |
| 4266 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 4267 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); |
| 4268 | const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 4269 | const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1)((__m512)__builtin_ia32_insertf32x8((__v16sf)(__m512)(_mm512_castps256_ps512 (row_scale_f32_ymm)), (__v8sf)(__m256)(row_scale_f32_ymm), (int )(1))); |
| 4270 | |
| 4271 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 4272 | acc_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 4273 | acc_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 4274 | acc_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 4275 | acc_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 4276 | |
| 4277 | // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K |
| 4278 | __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 4279 | __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 4280 | __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 4281 | __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 4282 | |
| 4283 | __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)85))), mins_23); |
| 4284 | __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)255))), mins_23); |
| 4285 | __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)85))), mins_23); |
| 4286 | __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)255))), mins_23); |
| 4287 | |
| 4288 | __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)0))), mins_45); |
| 4289 | __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)170))), mins_45); |
| 4290 | __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)0))), mins_45); |
| 4291 | __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)170))), mins_45); |
| 4292 | |
| 4293 | __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)85))), mins_67); |
| 4294 | __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)255))), mins_67); |
| 4295 | __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)85))), mins_67); |
| 4296 | __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)255))), mins_67); |
| 4297 | |
| 4298 | __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67)); |
| 4299 | __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67)); |
| 4300 | __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67)); |
| 4301 | __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67)); |
| 4302 | |
| 4303 | acc_min_rows[rp * 4] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_min_rows[rp * 4]); |
| 4304 | acc_min_rows[rp * 4 + 1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_min_rows[rp * 4 + 1]); |
| 4305 | acc_min_rows[rp * 4 + 2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_min_rows[rp * 4 + 2]); |
| 4306 | acc_min_rows[rp * 4 + 3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_min_rows[rp * 4 + 3]); |
| 4307 | } |
| 4308 | } |
| 4309 | } |
| 4310 | // Store the accumulated values |
| 4311 | for (int i = 0; i < 16; i++) { |
| 4312 | _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 4313 | } |
| 4314 | } |
| 4315 | } |
| 4316 | |
| 4317 | for (; y < nr / 4; y ++) { |
| 4318 | |
| 4319 | const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); |
| 4320 | |
| 4321 | // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation |
| 4322 | for (int64_t x = 0; x < anc / 8; x += 2) { |
| 4323 | |
| 4324 | const block_q2_Kx8 * b_ptr_0 = b_ptr_start + ((x) * b_nb); |
| 4325 | const block_q2_Kx8 * b_ptr_1 = b_ptr_start + ((x + 1) * b_nb); |
| 4326 | |
| 4327 | // Master FP accumulators |
| 4328 | __m512 acc_rows[4]; |
| 4329 | for (int i = 0; i < 4; i++) { |
| 4330 | acc_rows[i] = _mm512_setzero_ps(); |
| 4331 | } |
| 4332 | |
| 4333 | __m512 acc_min_rows[4]; |
| 4334 | for (int i = 0; i < 4; i++) { |
| 4335 | acc_min_rows[i] = _mm512_setzero_ps(); |
| 4336 | } |
| 4337 | // For super block |
| 4338 | for (int64_t b = 0; b < nb; b++) { |
| 4339 | // Delta values - Load the sixteen scale values from two block_q2_kx8 structures |
| 4340 | const __m512 col_scale_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].d, b_ptr_1[b].d); |
| 4341 | |
| 4342 | // dmin values - Load the sixteen dmin values from two block_q2_kx8 structures |
| 4343 | const __m512 col_dmin_f32 = GGML_F32Cx8x2_LOAD(b_ptr_0[b].dmin, b_ptr_1[b].dmin); |
| 4344 | |
| 4345 | // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration |
| 4346 | for (int sb = 0; sb < QK_K256 / 128; sb++) { |
| 4347 | |
| 4348 | // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 4349 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + sb * 256)); |
| 4350 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 32 + sb * 256)); |
| 4351 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 64 + sb * 256)); |
| 4352 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 96 + sb * 256)); |
| 4353 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 128 + sb * 256)); |
| 4354 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 160 + sb * 256)); |
| 4355 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 192 + sb * 256)); |
| 4356 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_0[b].qs + 224 + sb * 256)); |
| 4357 | |
| 4358 | const __m256i rhs_raw_mat_89AB_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + sb * 256)); |
| 4359 | const __m256i rhs_raw_mat_CDEF_0 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 32 + sb * 256)); |
| 4360 | const __m256i rhs_raw_mat_89AB_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 64 + sb * 256)); |
| 4361 | const __m256i rhs_raw_mat_CDEF_1 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 96 + sb * 256)); |
| 4362 | const __m256i rhs_raw_mat_89AB_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 128 + sb * 256)); |
| 4363 | const __m256i rhs_raw_mat_CDEF_2 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 160 + sb * 256)); |
| 4364 | const __m256i rhs_raw_mat_89AB_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 192 + sb * 256)); |
| 4365 | const __m256i rhs_raw_mat_CDEF_3 = _mm256_loadu_si256((const __m256i * )(b_ptr_1[b].qs + 224 + sb * 256)); |
| 4366 | |
| 4367 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 4368 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 4369 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 4370 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 4371 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 4372 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 4373 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 4374 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 4375 | |
| 4376 | const __m256i rhs_raw_mat_89CD_0 = _mm256_blend_epi32(rhs_raw_mat_89AB_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_0 , requiredOrder)), (int)(240))); |
| 4377 | const __m256i rhs_raw_mat_ABEF_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_0, requiredOrder), rhs_raw_mat_CDEF_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_0 ), (int)(240))); |
| 4378 | const __m256i rhs_raw_mat_89CD_1 = _mm256_blend_epi32(rhs_raw_mat_89AB_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_1 , requiredOrder)), (int)(240))); |
| 4379 | const __m256i rhs_raw_mat_ABEF_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_1, requiredOrder), rhs_raw_mat_CDEF_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_1 ), (int)(240))); |
| 4380 | const __m256i rhs_raw_mat_89CD_2 = _mm256_blend_epi32(rhs_raw_mat_89AB_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_2 , requiredOrder)), (int)(240))); |
| 4381 | const __m256i rhs_raw_mat_ABEF_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_2, requiredOrder), rhs_raw_mat_CDEF_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_2 ), (int)(240))); |
| 4382 | const __m256i rhs_raw_mat_89CD_3 = _mm256_blend_epi32(rhs_raw_mat_89AB_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_89AB_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_CDEF_3 , requiredOrder)), (int)(240))); |
| 4383 | const __m256i rhs_raw_mat_ABEF_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_89AB_3, requiredOrder), rhs_raw_mat_CDEF_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_89AB_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_CDEF_3 ), (int)(240))); |
| 4384 | |
| 4385 | const __m512i rhs_raw_mat_014589CD_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_0), rhs_raw_mat_89CD_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_0)), (__v8si)(__m256i)(rhs_raw_mat_89CD_0), (int)(1))); |
| 4386 | const __m512i rhs_raw_mat_2367ABEF_0 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_0), rhs_raw_mat_ABEF_0, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_0)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_0), (int)(1))); |
| 4387 | const __m512i rhs_raw_mat_014589CD_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_1), rhs_raw_mat_89CD_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_1)), (__v8si)(__m256i)(rhs_raw_mat_89CD_1), (int)(1))); |
| 4388 | const __m512i rhs_raw_mat_2367ABEF_1 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_1), rhs_raw_mat_ABEF_1, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_1)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_1), (int)(1))); |
| 4389 | |
| 4390 | const __m512i rhs_raw_mat_014589CD_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_2), rhs_raw_mat_89CD_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_2)), (__v8si)(__m256i)(rhs_raw_mat_89CD_2), (int)(1))); |
| 4391 | const __m512i rhs_raw_mat_2367ABEF_2 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_2), rhs_raw_mat_ABEF_2, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_2)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_2), (int)(1))); |
| 4392 | const __m512i rhs_raw_mat_014589CD_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_0145_3), rhs_raw_mat_89CD_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_0145_3)), (__v8si)(__m256i)(rhs_raw_mat_89CD_3), (int)(1))); |
| 4393 | const __m512i rhs_raw_mat_2367ABEF_3 = _mm512_inserti32x8(_mm512_castsi256_si512(rhs_raw_mat_2367_3), rhs_raw_mat_ABEF_3, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (rhs_raw_mat_2367_3)), (__v8si)(__m256i)(rhs_raw_mat_ABEF_3), (int)(1))); |
| 4394 | |
| 4395 | //2-bit -> 8-bit |
| 4396 | const __m512i rhs_mat_014589CD_00 = _mm512_and_si512(rhs_raw_mat_014589CD_0,m3bexpanded); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) B08(0-7) B09(0-7) B0C(0-7) B0D(0-7) |
| 4397 | const __m512i rhs_mat_2367ABEF_00 = _mm512_and_si512(rhs_raw_mat_2367ABEF_0,m3bexpanded); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) B0A(0-7) B0B(0-7) B0E(0-7) B0F(0-7) |
| 4398 | const __m512i rhs_mat_014589CD_01 = _mm512_and_si512(rhs_raw_mat_014589CD_1,m3bexpanded); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) B08(8-15) B09(8-15) B0C(8-15) B0D(8-15) |
| 4399 | const __m512i rhs_mat_2367ABEF_01 = _mm512_and_si512(rhs_raw_mat_2367ABEF_1,m3bexpanded); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) B0A(8-15) B0B(8-15) B0E(8-15) B0F(8-15) |
| 4400 | const __m512i rhs_mat_014589CD_10 = _mm512_and_si512(rhs_raw_mat_014589CD_2,m3bexpanded); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) B18(0-7) B19(0-7) B1C(0-7) B1D(0-7) |
| 4401 | const __m512i rhs_mat_2367ABEF_10 = _mm512_and_si512(rhs_raw_mat_2367ABEF_2,m3bexpanded); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) B1A(0-7) B1B(0-7) B1E(0-7) B1F(0-7) |
| 4402 | const __m512i rhs_mat_014589CD_11 = _mm512_and_si512(rhs_raw_mat_014589CD_3,m3bexpanded); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) B18(8-15) B19(8-15) B1C(8-15) B1D(8-15) |
| 4403 | const __m512i rhs_mat_2367ABEF_11 = _mm512_and_si512(rhs_raw_mat_2367ABEF_3,m3bexpanded); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) B1A(8-15) B1B(8-15) B1E(8-15) B1F(8-15) |
| 4404 | |
| 4405 | const __m512i rhs_mat_014589CD_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 2), m3bexpanded); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) B28(0-7) B29(0-7) B2C(0-7) B2D(0-7) |
| 4406 | const __m512i rhs_mat_2367ABEF_20 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 2), m3bexpanded); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) B2A(0-7) B2B(0-7) B2E(0-7) B2F(0-7) |
| 4407 | |
| 4408 | const __m512i rhs_mat_014589CD_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 2), m3bexpanded); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) B28(8-15) B29(8-15) B2C(8-15) B2D(8-15) |
| 4409 | const __m512i rhs_mat_2367ABEF_21 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 2), m3bexpanded); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) B2A(8-15) B2B(8-15) B2E(8-15) B2F(8-15) |
| 4410 | |
| 4411 | const __m512i rhs_mat_014589CD_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 2), m3bexpanded); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) B38(0-7) B39(0-7) B3C(0-7) B3D(0-7) |
| 4412 | const __m512i rhs_mat_2367ABEF_30 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 2), m3bexpanded); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) B3A(0-7) B3B(0-7) B3E(0-7) B3F(0-7) |
| 4413 | |
| 4414 | const __m512i rhs_mat_014589CD_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 2), m3bexpanded); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) B38(8-15) B39(8-15) B3C(8-15) B3D(8-15) |
| 4415 | const __m512i rhs_mat_2367ABEF_31 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 2), m3bexpanded); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) B3A(8-15) B3B(8-15) B3E(8-15) B3F(8-15) |
| 4416 | |
| 4417 | const __m512i rhs_mat_014589CD_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 4), m3bexpanded); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) B48(0-7) B49(0-7) B4C(0-7) B4D(0-7) |
| 4418 | const __m512i rhs_mat_2367ABEF_40 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 4), m3bexpanded); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) B4A(0-7) B4B(0-7) B4E(0-7) B4F(0-7) |
| 4419 | |
| 4420 | const __m512i rhs_mat_014589CD_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 4), m3bexpanded); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) B48(8-15) B49(8-15) B4C(8-15) B4D(8-15) |
| 4421 | const __m512i rhs_mat_2367ABEF_41 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 4), m3bexpanded); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) B4A(8-15) B4B(8-15) B4E(8-15) B4F(8-15) |
| 4422 | |
| 4423 | const __m512i rhs_mat_014589CD_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 4), m3bexpanded); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) B58(0-7) B59(0-7) B5C(0-7) B5D(0-7) |
| 4424 | const __m512i rhs_mat_2367ABEF_50 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 4), m3bexpanded); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) B5A(0-7) B5B(0-7) B5E(0-7) B5F(0-7) |
| 4425 | |
| 4426 | const __m512i rhs_mat_014589CD_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 4), m3bexpanded); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) B58(8-15) B59(8-15) B5C(8-15) B5D(8-15) |
| 4427 | const __m512i rhs_mat_2367ABEF_51 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 4), m3bexpanded); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) B5A(8-15) B5B(8-15) B5E(8-15) B5F(8-15) |
| 4428 | |
| 4429 | const __m512i rhs_mat_014589CD_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_0, 6), m3bexpanded); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) B68(0-7) B69(0-7) B6C(0-7) B6D(0-7) |
| 4430 | const __m512i rhs_mat_2367ABEF_60 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_0, 6), m3bexpanded); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) B6A(0-7) B6B(0-7) B6E(0-7) B6F(0-7) |
| 4431 | |
| 4432 | const __m512i rhs_mat_014589CD_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_1, 6), m3bexpanded); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) B68(8-15) B69(8-15) B6C(8-15) B6D(8-15) |
| 4433 | const __m512i rhs_mat_2367ABEF_61 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_1, 6), m3bexpanded); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) B6A(8-15) B6B(8-15) B6E(8-15) B6F(8-15) |
| 4434 | |
| 4435 | const __m512i rhs_mat_014589CD_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_2, 6), m3bexpanded); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) B78(0-7) B79(0-7) B7C(0-7) B7D(0-7) |
| 4436 | const __m512i rhs_mat_2367ABEF_70 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_2, 6), m3bexpanded); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) B7A(0-7) B7B(0-7) B7E(0-7) B7F(0-7) |
| 4437 | |
| 4438 | const __m512i rhs_mat_014589CD_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_014589CD_3, 6), m3bexpanded); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) B78(8-15) B79(8-15) B7C(8-15) B7D(8-15) |
| 4439 | const __m512i rhs_mat_2367ABEF_71 = _mm512_and_si512(_mm512_srli_epi16(rhs_raw_mat_2367ABEF_3, 6), m3bexpanded); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) B7A(8-15) B7B(8-15) B7E(8-15) B7F(8-15) |
| 4440 | |
| 4441 | const __m512i rhs_mat_014589CD_00_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) B08(0-3) B09(0-3) B08(0-3) B09(0-3) B0C(0-3) B0D(0-3) B0C(0-3) B0D(0-3) |
| 4442 | const __m512i rhs_mat_2367ABEF_00_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) B0A(0-3) B0B(0-3) B0A(0-3) B0B(0-3) B0E(0-3) B0F(0-3) B0E(0-3) B0F(0-3) |
| 4443 | |
| 4444 | const __m512i rhs_mat_014589CD_01_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 4445 | const __m512i rhs_mat_2367ABEF_01_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) B0A(8-11) B0B(8-11) B0A(8-11) B0B(8-11) B0E(8-11) B0F(8-11) B0E(8-11) B0F(8-11) |
| 4446 | |
| 4447 | const __m512i rhs_mat_014589CD_10_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) B18(0-3) B19(0-3) B18(0-3) B19(0-3) B1C(0-3) B1D(0-3) B1C(0-3) B1D(0-3) |
| 4448 | const __m512i rhs_mat_2367ABEF_10_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) B1A(0-3) B1B(0-3) B1A(0-3) B1B(0-3) B1E(0-3) B1F(0-3) B1E(0-3) B1F(0-3) |
| 4449 | |
| 4450 | const __m512i rhs_mat_014589CD_11_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) B18(8-11) B19(8-11) B18(8-11) B19(8-11) B1C(8-11) B1D(8-11) B1C(8-11) B1D(8-11) |
| 4451 | const __m512i rhs_mat_2367ABEF_11_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) B1A(8-11) B1B(8-11) B1A(8-11) B1B(8-11) B1E(8-11) B1F(8-11) B1E(8-11) B1F(8-11) |
| 4452 | |
| 4453 | const __m512i rhs_mat_014589CD_20_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_20 ), (int)((_MM_PERM_ENUM)136))); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) B28(0-3) B29(0-3) B28(0-3) B29(0-3) B2C(0-3) B2D(0-3) B2C(0-3) B2D(0-3) |
| 4454 | const __m512i rhs_mat_2367ABEF_20_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_20 ), (int)((_MM_PERM_ENUM)136))); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) B2A(0-3) B2B(0-3) B2A(0-3) B2B(0-3) B2E(0-3) B2F(0-3) B2E(0-3) B2F(0-3) |
| 4455 | |
| 4456 | const __m512i rhs_mat_014589CD_21_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_21 ), (int)((_MM_PERM_ENUM)136))); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) B28(8-11) B29(8-11) B28(8-11) B29(8-11) B2C(8-11) B2D(8-11) B2C(8-11) B2D(8-11) |
| 4457 | const __m512i rhs_mat_2367ABEF_21_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_21 ), (int)((_MM_PERM_ENUM)136))); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) B2A(8-11) B2B(8-11) B2A(8-11) B2B(8-11) B2E(8-11) B2F(8-11) B2E(8-11) B2F(8-11) |
| 4458 | const __m512i rhs_mat_014589CD_30_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_30 ), (int)((_MM_PERM_ENUM)136))); ///B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) B38(0-3) B39(0-3) B38(0-3) B39(0-3) B3C(0-3) B3D(0-3) B3C(0-3) B3D(0-3) |
| 4459 | const __m512i rhs_mat_2367ABEF_30_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_30 ), (int)((_MM_PERM_ENUM)136))); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) B3A(0-3) B3B(0-3) B3A(0-3) B3B(0-3) B3E(0-3) B3F(0-3) B3E(0-3) B3F(0-3) |
| 4460 | |
| 4461 | const __m512i rhs_mat_014589CD_31_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_31 ), (int)((_MM_PERM_ENUM)136))); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11) B38(8-11) B39(8-11) B38(8-11) B39(8-11) B3C(8-11) B3D(8-11) B3C(8-11) B3D(8-11) |
| 4462 | const __m512i rhs_mat_2367ABEF_31_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_31 ), (int)((_MM_PERM_ENUM)136))); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) B3A(8-11) B3B(8-11) B3A(8-11) B3B(8-11) B3E(8-11) B3F(8-11) B3E(8-11) B3F(8-11) |
| 4463 | |
| 4464 | const __m512i rhs_mat_014589CD_40_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_40 ), (int)((_MM_PERM_ENUM)136))); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) B48(0-3) B49(0-3) B48(0-3) B49(0-3) B4C(0-3) B4D(0-3) B4C(0-3) B4D(0-3) |
| 4465 | const __m512i rhs_mat_2367ABEF_40_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_40 ), (int)((_MM_PERM_ENUM)136))); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) B4A(0-3) B4B(0-3) B4A(0-3) B4B(0-3) B4E(0-3) B4F(0-3) B4E(0-3) B4F(0-3) |
| 4466 | |
| 4467 | const __m512i rhs_mat_014589CD_41_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_41 ), (int)((_MM_PERM_ENUM)136))); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) B48(8-11) B49(8-11) B48(8-11) B49(8-11) B4C(8-11) B4D(8-11) B4C(8-11) B4D(8-11) |
| 4468 | const __m512i rhs_mat_2367ABEF_41_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_41 ), (int)((_MM_PERM_ENUM)136))); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) B4A(8-11) B4B(8-11) B4A(8-11) B4B(8-11) B4E(8-11) B4F(8-11) B4E(8-11) B4F(8-11) |
| 4469 | |
| 4470 | const __m512i rhs_mat_014589CD_50_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_50 ), (int)((_MM_PERM_ENUM)136))); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) B58(0-3) B59(0-3) B58(0-3) B59(0-3) B5C(0-3) B5D(0-3) B5C(0-3) B5D(0-3) |
| 4471 | const __m512i rhs_mat_2367ABEF_50_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_50 ), (int)((_MM_PERM_ENUM)136))); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) B5A(0-3) B5B(0-3) B5A(0-3) B5B(0-3) B5E(0-3) B5F(0-3) B5E(0-3) B5F(0-3) |
| 4472 | |
| 4473 | const __m512i rhs_mat_014589CD_51_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_51 ), (int)((_MM_PERM_ENUM)136))); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) B58(8-11) B59(8-11) B58(8-11) B59(8-11) B5C(8-11) B5D(8-11) B5C(8-11) B5D(8-11) |
| 4474 | const __m512i rhs_mat_2367ABEF_51_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_51 ), (int)((_MM_PERM_ENUM)136))); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) B5A(8-11) B5B(8-11) B5A(8-11) B5B(8-11) B5E(8-11) B5F(8-11) B5E(8-11) B5F(8-11) |
| 4475 | |
| 4476 | const __m512i rhs_mat_014589CD_60_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_60 ), (int)((_MM_PERM_ENUM)136))); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) B68(0-3) B69(0-3) B68(0-3) B69(0-3) B6C(0-3) B6D(0-3) B6C(0-3) B6D(0-3) |
| 4477 | const __m512i rhs_mat_2367ABEF_60_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_60 ), (int)((_MM_PERM_ENUM)136))); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) B6A(0-3) B6B(0-3) B6A(0-3) B6B(0-3) B6E(0-3) B6F(0-3) B6E(0-3) B6F(0-3) |
| 4478 | |
| 4479 | const __m512i rhs_mat_014589CD_61_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_61 ), (int)((_MM_PERM_ENUM)136))); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) B68(8-11) B69(8-11) B68(8-11) B69(8-11) B6C(8-11) B6D(8-11) B6C(8-11) B6D(8-11) |
| 4480 | const __m512i rhs_mat_2367ABEF_61_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_61 ), (int)((_MM_PERM_ENUM)136))); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) B6A(8-11) B6B(8-11) B6A(8-11) B6B(8-11) B6E(8-11) B6F(8-11) B6E(8-11) B6F(8-11) |
| 4481 | |
| 4482 | const __m512i rhs_mat_014589CD_70_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_70 ), (int)((_MM_PERM_ENUM)136))); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) B78(0-3) B79(0-3) B78(0-3) B79(0-3) B7C(0-3) B7D(0-3) B7C(0-3) B7D(0-3) |
| 4483 | const __m512i rhs_mat_2367ABEF_70_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_70 ), (int)((_MM_PERM_ENUM)136))); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) B7A(0-3) B7B(0-3) B7A(0-3) B7B(0-3) B7E(0-3) B7F(0-3) B7E(0-3) B7F(0-3) |
| 4484 | |
| 4485 | const __m512i rhs_mat_014589CD_71_sp1 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_71 ), (int)((_MM_PERM_ENUM)136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) B08(8-11) B09(8-11) B08(8-11) B09(8-11) B0C(8-11) B0D(8-11) B0C(8-11) B0D(8-11) |
| 4486 | const __m512i rhs_mat_2367ABEF_71_sp1 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)136)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_71 ), (int)((_MM_PERM_ENUM)136))); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) B7A(8-11) B7B(8-11) B7A(8-11) B7B(8-11) B7E(8-11) B7F(8-11) B7E(8-11) B7F(8-11) |
| 4487 | |
| 4488 | const __m512i rhs_mat_014589CD_00_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_00 ), (int)((_MM_PERM_ENUM)221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) B08(4-7) B09(4-7) B08(4-7) B09(4-7) B0C(4-7) B0D(4-7) B0C(4-7) B0D(4-7) |
| 4489 | const __m512i rhs_mat_2367ABEF_00_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_00, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_00 ), (int)((_MM_PERM_ENUM)221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) B0A(4-7) B0B(4-7) B0A(4-7) B0B(4-7) B0E(4-7) B0F(4-7) B0E(4-7) B0F(4-7) |
| 4490 | |
| 4491 | const __m512i rhs_mat_014589CD_01_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_01 ), (int)((_MM_PERM_ENUM)221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) B08(12-15) B09(12-15) B08(12-15) B09(12-15) B0C(12-15) B0D(12-15) B0C(12-15) B0D(12-15) |
| 4492 | const __m512i rhs_mat_2367ABEF_01_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_01, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_01 ), (int)((_MM_PERM_ENUM)221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) B0A(12-15) B0B(12-15) B0A(12-15) B0B(12-15) B0E(12-15) B0F(12-15) B0E(12-15) B0F(12-15) |
| 4493 | |
| 4494 | const __m512i rhs_mat_014589CD_10_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_10 ), (int)((_MM_PERM_ENUM)221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) B18(4-7) B19(4-7) B18(4-7) B19(4-7) B1C(4-7) B1D(4-7) B1C(4-7) B1D(4-7) |
| 4495 | const __m512i rhs_mat_2367ABEF_10_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_10, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_10 ), (int)((_MM_PERM_ENUM)221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) B1A(4-7) B1B(4-7) B1A(4-7) B1B(4-7) B1E(4-7) B1F(4-7) B1E(4-7) B1F(4-7) |
| 4496 | |
| 4497 | const __m512i rhs_mat_014589CD_11_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_11 ), (int)((_MM_PERM_ENUM)221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) B18(12-15) B19(12-15) B18(12-15) B19(12-15) B1C(12-15) B1D(12-15) B1C(12-15) B1D(12-15) |
| 4498 | const __m512i rhs_mat_2367ABEF_11_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_11, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_11 ), (int)((_MM_PERM_ENUM)221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) B1A(12-15) B1B(12-15) B1A(12-15) B1B(12-15) B1E(12-15) B1F(12-15) B1E(12-15) B1F(12-15) |
| 4499 | |
| 4500 | const __m512i rhs_mat_014589CD_20_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_20, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_20 ), (int)((_MM_PERM_ENUM)221))); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) B28(4-7) B29(4-7) B28(4-7) B29(4-7) B2C(4-7) B2D(4-7) B2C(4-7) B2D(4-7) |
| 4501 | const __m512i rhs_mat_2367ABEF_20_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_20, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_20 ), (int)((_MM_PERM_ENUM)221))); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) B2A(4-7) B2B(4-7) B2A(4-7) B2B(4-7) B2E(4-7) B2F(4-7) B2E(4-7) B2F(4-7) |
| 4502 | |
| 4503 | const __m512i rhs_mat_014589CD_21_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_21, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_21 ), (int)((_MM_PERM_ENUM)221))); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) B28(12-15) B29(12-15) B28(12-15) B29(12-15) B2C(12-15) B2D(12-15) B2C(12-15) B2D(12-15) |
| 4504 | const __m512i rhs_mat_2367ABEF_21_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_21, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_21 ), (int)((_MM_PERM_ENUM)221))); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) B2A(12-15) B2B(12-15) B2A(12-15) B2B(12-15) B2E(12-15) B2F(12-15) B2E(12-15) B2F(12-15) |
| 4505 | |
| 4506 | const __m512i rhs_mat_014589CD_30_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_30, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_30 ), (int)((_MM_PERM_ENUM)221))); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) B38(4-7) B39(4-7) B38(4-7) B39(4-7) B3C(4-7) B3D(4-7) B3C(4-7) B3D(4-7) |
| 4507 | const __m512i rhs_mat_2367ABEF_30_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_30, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_30 ), (int)((_MM_PERM_ENUM)221))); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) B3A(4-7) B3B(4-7) B3A(4-7) B3B(4-7) B3E(4-7) B3F(4-7) B3E(4-7) B3F(4-7) |
| 4508 | |
| 4509 | const __m512i rhs_mat_014589CD_31_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_31, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_31 ), (int)((_MM_PERM_ENUM)221))); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) B38(12-15) B39(12-15) B38(12-15) B39(12-15) B3C(12-15) B3D(12-15) B3C(12-15) B3D(12-15) |
| 4510 | const __m512i rhs_mat_2367ABEF_31_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_31, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_31 ), (int)((_MM_PERM_ENUM)221))); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) B3A(12-15) B3B(12-15) B3A(12-15) B3B(12-15) B3E(12-15) B3F(12-15) B3E(12-15) B3F(12-15) |
| 4511 | |
| 4512 | const __m512i rhs_mat_014589CD_40_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_40, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_40 ), (int)((_MM_PERM_ENUM)221))); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) B48(4-7) B49(4-7) B48(4-7) B49(4-7) B4C(4-7) B4D(4-7) B4C(4-7) B4D(4-7) |
| 4513 | const __m512i rhs_mat_2367ABEF_40_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_40, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_40 ), (int)((_MM_PERM_ENUM)221))); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) B4A(4-7) B4B(4-7) B4A(4-7) B4B(4-7) B4E(4-7) B4F(4-7) B4E(4-7) B4F(4-7) |
| 4514 | |
| 4515 | const __m512i rhs_mat_014589CD_41_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_41, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_41 ), (int)((_MM_PERM_ENUM)221))); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) B48(12-15) B49(12-15) B48(12-15) B49(12-15) B4C(12-15) B4D(12-15) B4C(12-15) B4D(12-15) |
| 4516 | const __m512i rhs_mat_2367ABEF_41_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_41, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_41 ), (int)((_MM_PERM_ENUM)221))); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) B4A(12-15) B4B(12-15) B4A(12-15) B4B(12-15) B4E(12-15) B4F(12-15) B4E(12-15) B4F(12-15) |
| 4517 | |
| 4518 | const __m512i rhs_mat_014589CD_50_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_50, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_50 ), (int)((_MM_PERM_ENUM)221))); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) B58(4-7) B59(4-7) B58(4-7) B59(4-7) B5C(4-7) B5D(4-7) B5C(4-7) B5D(4-7) |
| 4519 | const __m512i rhs_mat_2367ABEF_50_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_50, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_50 ), (int)((_MM_PERM_ENUM)221))); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) B5A(4-7) B5B(4-7) B5A(4-7) B5B(4-7) B5E(4-7) B5F(4-7) B5E(4-7) B5F(4-7) |
| 4520 | |
| 4521 | const __m512i rhs_mat_014589CD_51_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_51, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_51 ), (int)((_MM_PERM_ENUM)221))); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) B58(12-15) B59(12-15) B58(12-15) B59(12-15) B5C(12-15) B5D(12-15) B5C(12-15) B5D(12-15) |
| 4522 | const __m512i rhs_mat_2367ABEF_51_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_51, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_51 ), (int)((_MM_PERM_ENUM)221))); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) B5A(12-15) B5B(12-15) B5A(12-15) B5B(12-15) B5E(12-15) B5F(12-15) B5E(12-15) B5F(12-15) |
| 4523 | |
| 4524 | const __m512i rhs_mat_014589CD_60_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_60, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_60 ), (int)((_MM_PERM_ENUM)221))); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) B68(4-7) B69(4-7) B68(4-7) B69(4-7) B6C(4-7) B6D(4-7) B6C(4-7) B6D(4-7) |
| 4525 | const __m512i rhs_mat_2367ABEF_60_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_60, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_60 ), (int)((_MM_PERM_ENUM)221))); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) B6A(4-7) B6B(4-7) B6A(4-7) B6B(4-7) B6E(4-7) B6F(4-7) B6E(4-7) B6F(4-7) |
| 4526 | |
| 4527 | const __m512i rhs_mat_014589CD_61_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_61, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_61 ), (int)((_MM_PERM_ENUM)221))); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) B68(12-15) B69(12-15) B68(12-15) B69(12-15) B6C(12-15) B6D(12-15) B6C(12-15) B6D(12-15) |
| 4528 | const __m512i rhs_mat_2367ABEF_61_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_61, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_61 ), (int)((_MM_PERM_ENUM)221))); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) B6A(12-15) B6B(12-15) B6A(12-15) B6B(12-15) B6E(12-15) B6F(12-15) B6E(12-15) B6F(12-15) |
| 4529 | |
| 4530 | const __m512i rhs_mat_014589CD_70_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_70, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_70 ), (int)((_MM_PERM_ENUM)221))); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) B78(4-7) B79(4-7) B78(4-7) B79(4-7) B7C(4-7) B7D(4-7) B7C(4-7) B7D(4-7) |
| 4531 | const __m512i rhs_mat_2367ABEF_70_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_70, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_70 ), (int)((_MM_PERM_ENUM)221))); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) B7A(4-7) B7B(4-7) B7A(4-7) B7B(4-7) B7E(4-7) B7F(4-7) B7E(4-7) B7F(4-7) |
| 4532 | |
| 4533 | const __m512i rhs_mat_014589CD_71_sp2 = _mm512_shuffle_epi32(rhs_mat_014589CD_71, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_014589CD_71 ), (int)((_MM_PERM_ENUM)221))); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) B78(12-15) B79(12-15) B78(12-15) B79(12-15) B7C(12-15) B7D(12-15) B7C(12-15) B7D(12-15) |
| 4534 | const __m512i rhs_mat_2367ABEF_71_sp2 = _mm512_shuffle_epi32(rhs_mat_2367ABEF_71, (_MM_PERM_ENUM)221)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(rhs_mat_2367ABEF_71 ), (int)((_MM_PERM_ENUM)221))); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) B7A(12-15) B7B(12-15) B7A(12-15) B7B(12-15) B7E(12-15) B7F(12-15) B7E(12-15) B7F(12-15) |
| 4535 | |
| 4536 | //notation:superblock subblock |
| 4537 | //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71 |
| 4538 | |
| 4539 | const __m128i mins_and_scales_01_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + sb * 64)); |
| 4540 | const __m128i mins_and_scales_23_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 16 + sb * 64)); |
| 4541 | const __m128i mins_and_scales_45_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 32 + sb * 64)); |
| 4542 | const __m128i mins_and_scales_67_0 = _mm_loadu_si128((const __m128i *)(b_ptr_0[b].scales + 48 + sb * 64)); |
| 4543 | |
| 4544 | const __m128i mins_and_scales_01_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + sb * 64)); |
| 4545 | const __m128i mins_and_scales_23_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 16 + sb * 64)); |
| 4546 | const __m128i mins_and_scales_45_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 32 + sb * 64)); |
| 4547 | const __m128i mins_and_scales_67_1 = _mm_loadu_si128((const __m128i *)(b_ptr_1[b].scales + 48 + sb * 64)); |
| 4548 | |
| 4549 | // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop |
| 4550 | const __m256i mins_and_scales_01 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_01_0), mins_and_scales_01_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_01_0)), (__v4si)(__m128i )(mins_and_scales_01_1), (int)(1))); |
| 4551 | const __m256i mins_and_scales_23 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_23_0), mins_and_scales_23_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_23_0)), (__v4si)(__m128i )(mins_and_scales_23_1), (int)(1))); |
| 4552 | const __m256i mins_and_scales_45 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_45_0), mins_and_scales_45_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_45_0)), (__v4si)(__m128i )(mins_and_scales_45_1), (int)(1))); |
| 4553 | const __m256i mins_and_scales_67 = _mm256_insertf128_si256(_mm256_castsi128_si256(mins_and_scales_67_0), mins_and_scales_67_1, 1)((__m256i)__builtin_ia32_vinsertf128_si256((__v8si)(__m256i)( _mm256_castsi128_si256(mins_and_scales_67_0)), (__v4si)(__m128i )(mins_and_scales_67_1), (int)(1))); |
| 4554 | |
| 4555 | // Extract scales which is lower half from mins_and_scales |
| 4556 | const __m256i scales_01 = _mm256_and_si256(mins_and_scales_01, m4b); |
| 4557 | const __m256i scales_23 = _mm256_and_si256(mins_and_scales_23, m4b); |
| 4558 | const __m256i scales_45 = _mm256_and_si256(mins_and_scales_45, m4b); |
| 4559 | const __m256i scales_67 = _mm256_and_si256(mins_and_scales_67, m4b); |
| 4560 | |
| 4561 | // Extract mins which is upper half from mins_and_scales |
| 4562 | const __m512i mins_01 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_01, 4), m4b)); |
| 4563 | const __m512i mins_23 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_23, 4), m4b)); |
| 4564 | const __m512i mins_45 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_45, 4), m4b)); |
| 4565 | const __m512i mins_67 = _mm512_cvtepu8_epi16(_mm256_and_si256(_mm256_srli_epi16(mins_and_scales_67, 4), m4b)); |
| 4566 | |
| 4567 | const __m512i scales_0 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask1)); |
| 4568 | const __m512i scales_1 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_01, scalesmask2)); |
| 4569 | const __m512i scales_2 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask1)); |
| 4570 | const __m512i scales_3 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_23, scalesmask2)); |
| 4571 | const __m512i scales_4 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask1)); |
| 4572 | const __m512i scales_5 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_45, scalesmask2)); |
| 4573 | const __m512i scales_6 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask1)); |
| 4574 | const __m512i scales_7 = _mm512_cvtepu8_epi16(_mm256_shuffle_epi8(scales_67, scalesmask2)); |
| 4575 | |
| 4576 | const __m512i scale_014589CD_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)68))); |
| 4577 | const __m512i scale_2367ABEF_0 = _mm512_shuffle_epi32(scales_0, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_0 ), (int)((_MM_PERM_ENUM)238))); |
| 4578 | |
| 4579 | const __m512i scale_014589CD_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)68))); |
| 4580 | const __m512i scale_2367ABEF_1 = _mm512_shuffle_epi32(scales_1, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_1 ), (int)((_MM_PERM_ENUM)238))); |
| 4581 | |
| 4582 | const __m512i scale_014589CD_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_2 ), (int)((_MM_PERM_ENUM)68))); |
| 4583 | const __m512i scale_2367ABEF_2 = _mm512_shuffle_epi32(scales_2, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_2 ), (int)((_MM_PERM_ENUM)238))); |
| 4584 | |
| 4585 | const __m512i scale_014589CD_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_3 ), (int)((_MM_PERM_ENUM)68))); |
| 4586 | const __m512i scale_2367ABEF_3 = _mm512_shuffle_epi32(scales_3, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_3 ), (int)((_MM_PERM_ENUM)238))); |
| 4587 | |
| 4588 | const __m512i scale_014589CD_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_4 ), (int)((_MM_PERM_ENUM)68))); |
| 4589 | const __m512i scale_2367ABEF_4 = _mm512_shuffle_epi32(scales_4, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_4 ), (int)((_MM_PERM_ENUM)238))); |
| 4590 | |
| 4591 | const __m512i scale_014589CD_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_5 ), (int)((_MM_PERM_ENUM)68))); |
| 4592 | const __m512i scale_2367ABEF_5 = _mm512_shuffle_epi32(scales_5, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_5 ), (int)((_MM_PERM_ENUM)238))); |
| 4593 | |
| 4594 | const __m512i scale_014589CD_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_6 ), (int)((_MM_PERM_ENUM)68))); |
| 4595 | const __m512i scale_2367ABEF_6 = _mm512_shuffle_epi32(scales_6, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_6 ), (int)((_MM_PERM_ENUM)238))); |
| 4596 | |
| 4597 | const __m512i scale_014589CD_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)68)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_7 ), (int)((_MM_PERM_ENUM)68))); |
| 4598 | const __m512i scale_2367ABEF_7 = _mm512_shuffle_epi32(scales_7, (_MM_PERM_ENUM)238)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(scales_7 ), (int)((_MM_PERM_ENUM)238))); |
| 4599 | |
| 4600 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 4601 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 4602 | __m256i lhs_mat_ymm_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb))); |
| 4603 | __m256i lhs_mat_ymm_01_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(0))); |
| 4604 | __m256i lhs_mat_ymm_23_00 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_00, lhs_mat_ymm_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_00 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_00), (int)(17))); |
| 4605 | __m256i lhs_mat_ymm_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb))); |
| 4606 | __m256i lhs_mat_ymm_01_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(0))); |
| 4607 | __m256i lhs_mat_ymm_23_01 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_01, lhs_mat_ymm_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_01 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_01), (int)(17))); |
| 4608 | __m256i lhs_mat_ymm_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb))); |
| 4609 | __m256i lhs_mat_ymm_01_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(0))); |
| 4610 | __m256i lhs_mat_ymm_23_10 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_10, lhs_mat_ymm_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_10 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_10), (int)(17))); |
| 4611 | __m256i lhs_mat_ymm_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb))); |
| 4612 | __m256i lhs_mat_ymm_01_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(0))); |
| 4613 | __m256i lhs_mat_ymm_23_11 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_11, lhs_mat_ymm_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_11 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_11), (int)(17))); |
| 4614 | __m256i lhs_mat_ymm_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb))); |
| 4615 | __m256i lhs_mat_ymm_01_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_20 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_20), (int)(0))); |
| 4616 | __m256i lhs_mat_ymm_23_20 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_20, lhs_mat_ymm_0123_20, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_20 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_20), (int)(17))); |
| 4617 | __m256i lhs_mat_ymm_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb))); |
| 4618 | __m256i lhs_mat_ymm_01_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_21 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_21), (int)(0))); |
| 4619 | __m256i lhs_mat_ymm_23_21 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_21, lhs_mat_ymm_0123_21, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_21 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_21), (int)(17))); |
| 4620 | __m256i lhs_mat_ymm_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb))); |
| 4621 | __m256i lhs_mat_ymm_01_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_30 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_30), (int)(0))); |
| 4622 | __m256i lhs_mat_ymm_23_30 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_30, lhs_mat_ymm_0123_30, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_30 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_30), (int)(17))); |
| 4623 | __m256i lhs_mat_ymm_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb))); |
| 4624 | __m256i lhs_mat_ymm_01_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_31 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_31), (int)(0))); |
| 4625 | __m256i lhs_mat_ymm_23_31 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_31, lhs_mat_ymm_0123_31, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_31 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_31), (int)(17))); |
| 4626 | |
| 4627 | __m256i lhs_mat_ymm_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb))); |
| 4628 | __m256i lhs_mat_ymm_01_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_40 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_40), (int)(0))); |
| 4629 | __m256i lhs_mat_ymm_23_40 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_40, lhs_mat_ymm_0123_40, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_40 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_40), (int)(17))); |
| 4630 | __m256i lhs_mat_ymm_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb))); |
| 4631 | __m256i lhs_mat_ymm_01_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_41 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_41), (int)(0))); |
| 4632 | __m256i lhs_mat_ymm_23_41 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_41, lhs_mat_ymm_0123_41, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_41 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_41), (int)(17))); |
| 4633 | __m256i lhs_mat_ymm_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb))); |
| 4634 | __m256i lhs_mat_ymm_01_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_50 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_50), (int)(0))); |
| 4635 | __m256i lhs_mat_ymm_23_50 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_50, lhs_mat_ymm_0123_50, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_50 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_50), (int)(17))); |
| 4636 | __m256i lhs_mat_ymm_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb))); |
| 4637 | __m256i lhs_mat_ymm_01_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_51 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_51), (int)(0))); |
| 4638 | __m256i lhs_mat_ymm_23_51 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_51, lhs_mat_ymm_0123_51, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_51 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_51), (int)(17))); |
| 4639 | __m256i lhs_mat_ymm_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb))); |
| 4640 | __m256i lhs_mat_ymm_01_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_60 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_60), (int)(0))); |
| 4641 | __m256i lhs_mat_ymm_23_60 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_60, lhs_mat_ymm_0123_60, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_60 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_60), (int)(17))); |
| 4642 | __m256i lhs_mat_ymm_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb))); |
| 4643 | __m256i lhs_mat_ymm_01_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_61 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_61), (int)(0))); |
| 4644 | __m256i lhs_mat_ymm_23_61 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_61, lhs_mat_ymm_0123_61, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_61 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_61), (int)(17))); |
| 4645 | __m256i lhs_mat_ymm_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb))); |
| 4646 | __m256i lhs_mat_ymm_01_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_70 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_70), (int)(0))); |
| 4647 | __m256i lhs_mat_ymm_23_70 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_70, lhs_mat_ymm_0123_70, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_70 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_70), (int)(17))); |
| 4648 | __m256i lhs_mat_ymm_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb))); |
| 4649 | __m256i lhs_mat_ymm_01_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_71 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_71), (int)(0))); |
| 4650 | __m256i lhs_mat_ymm_23_71 = _mm256_permute2f128_si256(lhs_mat_ymm_0123_71, lhs_mat_ymm_0123_71, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_ymm_0123_71 ), (__v8si)(__m256i)(lhs_mat_ymm_0123_71), (int)(17))); |
| 4651 | |
| 4652 | __m512i lhs_mat_01_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_00), lhs_mat_ymm_01_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_00)), (__v8si)(__m256i)(lhs_mat_ymm_01_00), ( int)(1))); |
| 4653 | __m512i lhs_mat_23_00 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_00), lhs_mat_ymm_23_00, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_00)), (__v8si)(__m256i)(lhs_mat_ymm_23_00), ( int)(1))); |
| 4654 | __m512i lhs_mat_01_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_01), lhs_mat_ymm_01_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_01)), (__v8si)(__m256i)(lhs_mat_ymm_01_01), ( int)(1))); |
| 4655 | __m512i lhs_mat_23_01 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_01), lhs_mat_ymm_23_01, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_01)), (__v8si)(__m256i)(lhs_mat_ymm_23_01), ( int)(1))); |
| 4656 | |
| 4657 | __m512i lhs_mat_01_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_10), lhs_mat_ymm_01_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_10)), (__v8si)(__m256i)(lhs_mat_ymm_01_10), ( int)(1))); |
| 4658 | __m512i lhs_mat_23_10 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_10), lhs_mat_ymm_23_10, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_10)), (__v8si)(__m256i)(lhs_mat_ymm_23_10), ( int)(1))); |
| 4659 | __m512i lhs_mat_01_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_11), lhs_mat_ymm_01_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_11)), (__v8si)(__m256i)(lhs_mat_ymm_01_11), ( int)(1))); |
| 4660 | __m512i lhs_mat_23_11 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_11), lhs_mat_ymm_23_11, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_11)), (__v8si)(__m256i)(lhs_mat_ymm_23_11), ( int)(1))); |
| 4661 | |
| 4662 | __m512i lhs_mat_01_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_20), lhs_mat_ymm_01_20, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_20)), (__v8si)(__m256i)(lhs_mat_ymm_01_20), ( int)(1))); |
| 4663 | __m512i lhs_mat_23_20 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_20), lhs_mat_ymm_23_20, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_20)), (__v8si)(__m256i)(lhs_mat_ymm_23_20), ( int)(1))); |
| 4664 | __m512i lhs_mat_01_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_21), lhs_mat_ymm_01_21, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_21)), (__v8si)(__m256i)(lhs_mat_ymm_01_21), ( int)(1))); |
| 4665 | __m512i lhs_mat_23_21 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_21), lhs_mat_ymm_23_21, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_21)), (__v8si)(__m256i)(lhs_mat_ymm_23_21), ( int)(1))); |
| 4666 | |
| 4667 | __m512i lhs_mat_01_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_30), lhs_mat_ymm_01_30, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_30)), (__v8si)(__m256i)(lhs_mat_ymm_01_30), ( int)(1))); |
| 4668 | __m512i lhs_mat_23_30 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_30), lhs_mat_ymm_23_30, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_30)), (__v8si)(__m256i)(lhs_mat_ymm_23_30), ( int)(1))); |
| 4669 | __m512i lhs_mat_01_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_31), lhs_mat_ymm_01_31, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_31)), (__v8si)(__m256i)(lhs_mat_ymm_01_31), ( int)(1))); |
| 4670 | __m512i lhs_mat_23_31 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_31), lhs_mat_ymm_23_31, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_31)), (__v8si)(__m256i)(lhs_mat_ymm_23_31), ( int)(1))); |
| 4671 | |
| 4672 | __m512i lhs_mat_01_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_40), lhs_mat_ymm_01_40, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_40)), (__v8si)(__m256i)(lhs_mat_ymm_01_40), ( int)(1))); |
| 4673 | __m512i lhs_mat_23_40 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_40), lhs_mat_ymm_23_40, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_40)), (__v8si)(__m256i)(lhs_mat_ymm_23_40), ( int)(1))); |
| 4674 | __m512i lhs_mat_01_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_41), lhs_mat_ymm_01_41, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_41)), (__v8si)(__m256i)(lhs_mat_ymm_01_41), ( int)(1))); |
| 4675 | __m512i lhs_mat_23_41 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_41), lhs_mat_ymm_23_41, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_41)), (__v8si)(__m256i)(lhs_mat_ymm_23_41), ( int)(1))); |
| 4676 | |
| 4677 | __m512i lhs_mat_01_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_50), lhs_mat_ymm_01_50, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_50)), (__v8si)(__m256i)(lhs_mat_ymm_01_50), ( int)(1))); |
| 4678 | __m512i lhs_mat_23_50 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_50), lhs_mat_ymm_23_50, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_50)), (__v8si)(__m256i)(lhs_mat_ymm_23_50), ( int)(1))); |
| 4679 | __m512i lhs_mat_01_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_51), lhs_mat_ymm_01_51, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_51)), (__v8si)(__m256i)(lhs_mat_ymm_01_51), ( int)(1))); |
| 4680 | __m512i lhs_mat_23_51 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_51), lhs_mat_ymm_23_51, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_51)), (__v8si)(__m256i)(lhs_mat_ymm_23_51), ( int)(1))); |
| 4681 | |
| 4682 | __m512i lhs_mat_01_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_60), lhs_mat_ymm_01_60, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_60)), (__v8si)(__m256i)(lhs_mat_ymm_01_60), ( int)(1))); |
| 4683 | __m512i lhs_mat_23_60 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_60), lhs_mat_ymm_23_60, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_60)), (__v8si)(__m256i)(lhs_mat_ymm_23_60), ( int)(1))); |
| 4684 | __m512i lhs_mat_01_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_61), lhs_mat_ymm_01_61, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_61)), (__v8si)(__m256i)(lhs_mat_ymm_01_61), ( int)(1))); |
| 4685 | __m512i lhs_mat_23_61 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_61), lhs_mat_ymm_23_61, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_61)), (__v8si)(__m256i)(lhs_mat_ymm_23_61), ( int)(1))); |
| 4686 | |
| 4687 | __m512i lhs_mat_01_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_70), lhs_mat_ymm_01_70, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_70)), (__v8si)(__m256i)(lhs_mat_ymm_01_70), ( int)(1))); |
| 4688 | __m512i lhs_mat_23_70 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_70), lhs_mat_ymm_23_70, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_70)), (__v8si)(__m256i)(lhs_mat_ymm_23_70), ( int)(1))); |
| 4689 | __m512i lhs_mat_01_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_01_71), lhs_mat_ymm_01_71, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_01_71)), (__v8si)(__m256i)(lhs_mat_ymm_01_71), ( int)(1))); |
| 4690 | __m512i lhs_mat_23_71 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_mat_ymm_23_71), lhs_mat_ymm_23_71, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_mat_ymm_23_71)), (__v8si)(__m256i)(lhs_mat_ymm_23_71), ( int)(1))); |
| 4691 | |
| 4692 | // Bsums are loaded for the different Q8_K blocks |
| 4693 | __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb))); |
| 4694 | __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb)); |
| 4695 | __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb))); |
| 4696 | __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb)); |
| 4697 | |
| 4698 | __m256i lhs_bsums_ymm_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_0123)), (__v2di)(__m128i)(lhs_raw_bsums_01_0123 ), (int)(1))); |
| 4699 | __m512i lhs_bsums_01_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_0123), lhs_bsums_ymm_01_0123, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_01_0123)), (__v8si)(__m256i)(lhs_bsums_ymm_01_0123 ), (int)(1))); |
| 4700 | __m256i lhs_bsums_ymm_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_0123)), (__v2di)(__m128i)(lhs_raw_bsums_23_0123 ), (int)(1))); |
| 4701 | __m512i lhs_bsums_23_0123 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_0123), lhs_bsums_ymm_23_0123, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_23_0123)), (__v8si)(__m256i)(lhs_bsums_ymm_23_0123 ), (int)(1))); |
| 4702 | __m256i lhs_bsums_ymm_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_4567)), (__v2di)(__m128i)(lhs_raw_bsums_01_4567 ), (int)(1))); |
| 4703 | __m512i lhs_bsums_01_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_01_4567), lhs_bsums_ymm_01_4567, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_01_4567)), (__v8si)(__m256i)(lhs_bsums_ymm_01_4567 ), (int)(1))); |
| 4704 | __m256i lhs_bsums_ymm_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_4567)), (__v2di)(__m128i)(lhs_raw_bsums_23_4567 ), (int)(1))); |
| 4705 | __m512i lhs_bsums_23_4567 = _mm512_inserti32x8(_mm512_castsi256_si512(lhs_bsums_ymm_23_4567), lhs_bsums_ymm_23_4567, 1)((__m512i)__builtin_ia32_inserti32x8((__v16si)(__m512i)(_mm512_castsi256_si512 (lhs_bsums_ymm_23_4567)), (__v8si)(__m256i)(lhs_bsums_ymm_23_4567 ), (int)(1))); |
| 4706 | |
| 4707 | // Shuffle pattern one - left side input |
| 4708 | const __m512i lhs_mat_01_00_sp1 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 4709 | const __m512i lhs_mat_23_00_sp1 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)160))); //A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) A02(0-3) A02(0-3) A03(0-3) A03(0-3) |
| 4710 | |
| 4711 | const __m512i lhs_mat_01_01_sp1 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 4712 | const __m512i lhs_mat_23_01_sp1 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)160))); //A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) A02(8-11) A02(8-11) A03(8-11) A03(8-11) |
| 4713 | |
| 4714 | const __m512i lhs_mat_01_10_sp1 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 4715 | const __m512i lhs_mat_23_10_sp1 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)160))); //A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) A12(0-3) A12(0-3) A13(0-3) A13(0-3) |
| 4716 | |
| 4717 | const __m512i lhs_mat_01_11_sp1 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 4718 | const __m512i lhs_mat_23_11_sp1 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)160))); //A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) A12(8-11) A12(8-11) A13(8-11) A13(8-11) |
| 4719 | |
| 4720 | const __m512i lhs_mat_01_20_sp1 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_20 ), (int)((_MM_PERM_ENUM)160))); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) |
| 4721 | const __m512i lhs_mat_23_20_sp1 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_20 ), (int)((_MM_PERM_ENUM)160))); //A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) A22(0-3) A22(0-3) A23(0-3) A23(0-3) |
| 4722 | |
| 4723 | const __m512i lhs_mat_01_21_sp1 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_21 ), (int)((_MM_PERM_ENUM)160))); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) |
| 4724 | const __m512i lhs_mat_23_21_sp1 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_21 ), (int)((_MM_PERM_ENUM)160))); //A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) A22(8-11) A22(8-11) A23(8-11) A23(8-11) |
| 4725 | |
| 4726 | const __m512i lhs_mat_01_30_sp1 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_30 ), (int)((_MM_PERM_ENUM)160))); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) |
| 4727 | const __m512i lhs_mat_23_30_sp1 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_30 ), (int)((_MM_PERM_ENUM)160))); //A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) A32(0-3) A32(0-3) A33(0-3) A33(0-3) |
| 4728 | |
| 4729 | const __m512i lhs_mat_01_31_sp1 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_31 ), (int)((_MM_PERM_ENUM)160))); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) |
| 4730 | const __m512i lhs_mat_23_31_sp1 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_31 ), (int)((_MM_PERM_ENUM)160))); //A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) A32(8-11) A32(8-11) A33(8-11) A33(8-11) |
| 4731 | |
| 4732 | const __m512i lhs_mat_01_40_sp1 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_40 ), (int)((_MM_PERM_ENUM)160))); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) |
| 4733 | const __m512i lhs_mat_23_40_sp1 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_40 ), (int)((_MM_PERM_ENUM)160))); //A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) A42(0-3) A42(0-3) A43(0-3) A43(0-3) |
| 4734 | |
| 4735 | const __m512i lhs_mat_01_41_sp1 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_41 ), (int)((_MM_PERM_ENUM)160))); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) |
| 4736 | const __m512i lhs_mat_23_41_sp1 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_41 ), (int)((_MM_PERM_ENUM)160))); //A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) A42(8-11) A42(8-11) A43(8-11) A43(8-11) |
| 4737 | |
| 4738 | const __m512i lhs_mat_01_50_sp1 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_50 ), (int)((_MM_PERM_ENUM)160))); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) |
| 4739 | const __m512i lhs_mat_23_50_sp1 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_50 ), (int)((_MM_PERM_ENUM)160))); //A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) A52(0-3) A52(0-3) A53(0-3) A53(0-3) |
| 4740 | |
| 4741 | const __m512i lhs_mat_01_51_sp1 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_51 ), (int)((_MM_PERM_ENUM)160))); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) |
| 4742 | const __m512i lhs_mat_23_51_sp1 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_51 ), (int)((_MM_PERM_ENUM)160))); //A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) A52(8-11) A52(8-11) A53(8-11) A53(8-11) |
| 4743 | |
| 4744 | const __m512i lhs_mat_01_60_sp1 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_60 ), (int)((_MM_PERM_ENUM)160))); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) |
| 4745 | const __m512i lhs_mat_23_60_sp1 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_60 ), (int)((_MM_PERM_ENUM)160))); //A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) A62(0-3) A62(0-3) A63(0-3) A63(0-3) |
| 4746 | |
| 4747 | const __m512i lhs_mat_01_61_sp1 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_61 ), (int)((_MM_PERM_ENUM)160))); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) |
| 4748 | const __m512i lhs_mat_23_61_sp1 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_61 ), (int)((_MM_PERM_ENUM)160))); //A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) A62(8-11) A62(8-11) A63(8-11) A63(8-11) |
| 4749 | |
| 4750 | const __m512i lhs_mat_01_70_sp1 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_70 ), (int)((_MM_PERM_ENUM)160))); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) |
| 4751 | const __m512i lhs_mat_23_70_sp1 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_70 ), (int)((_MM_PERM_ENUM)160))); //A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) A72(0-3) A72(0-3) A73(0-3) A73(0-3) |
| 4752 | |
| 4753 | const __m512i lhs_mat_01_71_sp1 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_71 ), (int)((_MM_PERM_ENUM)160))); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) |
| 4754 | const __m512i lhs_mat_23_71_sp1 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)160)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_71 ), (int)((_MM_PERM_ENUM)160))); //A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) A72(8-11) A72(8-11) A73(8-11) A73(8-11) |
| 4755 | |
| 4756 | const __m512i lhs_mat_01_00_sp2 = _mm512_shuffle_epi32(lhs_mat_01_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_00 ), (int)((_MM_PERM_ENUM)245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 4757 | const __m512i lhs_mat_23_00_sp2 = _mm512_shuffle_epi32(lhs_mat_23_00, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_00 ), (int)((_MM_PERM_ENUM)245))); //A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) A02(4-7) A02(4-7) A03(4-7) A03(4-7) |
| 4758 | |
| 4759 | const __m512i lhs_mat_01_01_sp2 = _mm512_shuffle_epi32(lhs_mat_01_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_01 ), (int)((_MM_PERM_ENUM)245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 4760 | const __m512i lhs_mat_23_01_sp2 = _mm512_shuffle_epi32(lhs_mat_23_01, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_01 ), (int)((_MM_PERM_ENUM)245))); //A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) A02(12-15) A02(12-15) A03(12-15) A03(12-15) |
| 4761 | |
| 4762 | const __m512i lhs_mat_01_10_sp2 = _mm512_shuffle_epi32(lhs_mat_01_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_10 ), (int)((_MM_PERM_ENUM)245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 4763 | const __m512i lhs_mat_23_10_sp2 = _mm512_shuffle_epi32(lhs_mat_23_10, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_10 ), (int)((_MM_PERM_ENUM)245))); //A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) A12(4-7) A12(4-7) A13(4-7) A13(4-7) |
| 4764 | |
| 4765 | const __m512i lhs_mat_01_11_sp2 = _mm512_shuffle_epi32(lhs_mat_01_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_11 ), (int)((_MM_PERM_ENUM)245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 4766 | const __m512i lhs_mat_23_11_sp2 = _mm512_shuffle_epi32(lhs_mat_23_11, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_11 ), (int)((_MM_PERM_ENUM)245))); //A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) A12(12-15) A12(12-15) A13(12-15) A13(12-15) |
| 4767 | |
| 4768 | const __m512i lhs_mat_01_20_sp2 = _mm512_shuffle_epi32(lhs_mat_01_20, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_20 ), (int)((_MM_PERM_ENUM)245))); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) |
| 4769 | const __m512i lhs_mat_23_20_sp2 = _mm512_shuffle_epi32(lhs_mat_23_20, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_20 ), (int)((_MM_PERM_ENUM)245))); //A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) A22(4-7) A22(4-7) A23(4-7) A23(4-7) |
| 4770 | |
| 4771 | const __m512i lhs_mat_01_21_sp2 = _mm512_shuffle_epi32(lhs_mat_01_21, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_21 ), (int)((_MM_PERM_ENUM)245))); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) |
| 4772 | const __m512i lhs_mat_23_21_sp2 = _mm512_shuffle_epi32(lhs_mat_23_21, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_21 ), (int)((_MM_PERM_ENUM)245))); //A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) A22(12-15) A22(12-15) A23(12-15) A23(12-15) |
| 4773 | |
| 4774 | const __m512i lhs_mat_01_30_sp2 = _mm512_shuffle_epi32(lhs_mat_01_30, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_30 ), (int)((_MM_PERM_ENUM)245))); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) |
| 4775 | const __m512i lhs_mat_23_30_sp2 = _mm512_shuffle_epi32(lhs_mat_23_30, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_30 ), (int)((_MM_PERM_ENUM)245))); //A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) A32(4-7) A32(4-7) A33(4-7) A33(4-7) |
| 4776 | |
| 4777 | const __m512i lhs_mat_01_31_sp2 = _mm512_shuffle_epi32(lhs_mat_01_31, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_31 ), (int)((_MM_PERM_ENUM)245))); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) |
| 4778 | const __m512i lhs_mat_23_31_sp2 = _mm512_shuffle_epi32(lhs_mat_23_31, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_31 ), (int)((_MM_PERM_ENUM)245))); //A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) A32(12-15) A32(12-15) A33(12-15) A33(12-15) |
| 4779 | |
| 4780 | const __m512i lhs_mat_01_40_sp2 = _mm512_shuffle_epi32(lhs_mat_01_40, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_40 ), (int)((_MM_PERM_ENUM)245))); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) |
| 4781 | const __m512i lhs_mat_23_40_sp2 = _mm512_shuffle_epi32(lhs_mat_23_40, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_40 ), (int)((_MM_PERM_ENUM)245))); //A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) A42(4-7) A42(4-7) A43(4-7) A43(4-7) |
| 4782 | |
| 4783 | const __m512i lhs_mat_01_41_sp2 = _mm512_shuffle_epi32(lhs_mat_01_41, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_41 ), (int)((_MM_PERM_ENUM)245))); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) |
| 4784 | const __m512i lhs_mat_23_41_sp2 = _mm512_shuffle_epi32(lhs_mat_23_41, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_41 ), (int)((_MM_PERM_ENUM)245))); //A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) A42(12-15) A42(12-15) A43(12-15) A43(12-15) |
| 4785 | |
| 4786 | const __m512i lhs_mat_01_50_sp2 = _mm512_shuffle_epi32(lhs_mat_01_50, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_50 ), (int)((_MM_PERM_ENUM)245))); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) |
| 4787 | const __m512i lhs_mat_23_50_sp2 = _mm512_shuffle_epi32(lhs_mat_23_50, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_50 ), (int)((_MM_PERM_ENUM)245))); //A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) A52(4-7) A52(4-7) A53(4-7) A53(4-7) |
| 4788 | |
| 4789 | const __m512i lhs_mat_01_51_sp2 = _mm512_shuffle_epi32(lhs_mat_01_51, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_51 ), (int)((_MM_PERM_ENUM)245))); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) |
| 4790 | const __m512i lhs_mat_23_51_sp2 = _mm512_shuffle_epi32(lhs_mat_23_51, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_51 ), (int)((_MM_PERM_ENUM)245))); //A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) A52(12-15) A52(12-15) A53(12-15) A53(12-15) |
| 4791 | |
| 4792 | const __m512i lhs_mat_01_60_sp2 = _mm512_shuffle_epi32(lhs_mat_01_60, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_60 ), (int)((_MM_PERM_ENUM)245))); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) |
| 4793 | const __m512i lhs_mat_23_60_sp2 = _mm512_shuffle_epi32(lhs_mat_23_60, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_60 ), (int)((_MM_PERM_ENUM)245))); //A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) A62(4-7) A62(4-7) A63(4-7) A63(4-7) |
| 4794 | |
| 4795 | const __m512i lhs_mat_01_61_sp2 = _mm512_shuffle_epi32(lhs_mat_01_61, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_61 ), (int)((_MM_PERM_ENUM)245))); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) |
| 4796 | const __m512i lhs_mat_23_61_sp2 = _mm512_shuffle_epi32(lhs_mat_23_61, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_61 ), (int)((_MM_PERM_ENUM)245))); //A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) A62(12-15) A62(12-15) A63(12-15) A63(12-15) |
| 4797 | |
| 4798 | const __m512i lhs_mat_01_70_sp2 = _mm512_shuffle_epi32(lhs_mat_01_70, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_70 ), (int)((_MM_PERM_ENUM)245))); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) |
| 4799 | const __m512i lhs_mat_23_70_sp2 = _mm512_shuffle_epi32(lhs_mat_23_70, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_70 ), (int)((_MM_PERM_ENUM)245))); //A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) A72(4-7) A72(4-7) A73(4-7) A73(4-7) |
| 4800 | |
| 4801 | const __m512i lhs_mat_01_71_sp2 = _mm512_shuffle_epi32(lhs_mat_01_71, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_01_71 ), (int)((_MM_PERM_ENUM)245))); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) |
| 4802 | const __m512i lhs_mat_23_71_sp2 = _mm512_shuffle_epi32(lhs_mat_23_71, (_MM_PERM_ENUM)245)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_mat_23_71 ), (int)((_MM_PERM_ENUM)245))); //A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) A72(12-15) A72(12-15) A73(12-15) A73(12-15) |
| 4803 | |
| 4804 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 4805 | __m512i iacc_mat_00_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_01_01_sp1)); |
| 4806 | __m512i iacc_mat_01_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_01_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_01_01_sp1)); |
| 4807 | |
| 4808 | __m512i iacc_mat_10_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp1, lhs_mat_23_01_sp1)); |
| 4809 | __m512i iacc_mat_11_0_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp1, lhs_mat_23_00_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp1, lhs_mat_23_01_sp1)); |
| 4810 | |
| 4811 | __m512i iacc_mat_00_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_01_11_sp1)); |
| 4812 | __m512i iacc_mat_01_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_01_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_01_11_sp1)); |
| 4813 | |
| 4814 | __m512i iacc_mat_10_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp1, lhs_mat_23_11_sp1)); |
| 4815 | __m512i iacc_mat_11_1_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp1, lhs_mat_23_10_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp1, lhs_mat_23_11_sp1)); |
| 4816 | |
| 4817 | __m512i iacc_mat_00_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_01_21_sp1)); |
| 4818 | __m512i iacc_mat_01_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_01_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_01_21_sp1)); |
| 4819 | |
| 4820 | __m512i iacc_mat_10_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp1, lhs_mat_23_21_sp1)); |
| 4821 | __m512i iacc_mat_11_2_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp1, lhs_mat_23_20_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp1, lhs_mat_23_21_sp1)); |
| 4822 | |
| 4823 | __m512i iacc_mat_00_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_01_31_sp1)); |
| 4824 | __m512i iacc_mat_01_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_01_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_01_31_sp1)); |
| 4825 | |
| 4826 | __m512i iacc_mat_10_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp1, lhs_mat_23_31_sp1)); |
| 4827 | __m512i iacc_mat_11_3_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp1, lhs_mat_23_30_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp1, lhs_mat_23_31_sp1)); |
| 4828 | |
| 4829 | __m512i iacc_mat_00_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_01_41_sp1)); |
| 4830 | __m512i iacc_mat_01_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_01_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_01_41_sp1)); |
| 4831 | |
| 4832 | __m512i iacc_mat_10_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp1, lhs_mat_23_41_sp1)); |
| 4833 | __m512i iacc_mat_11_4_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp1, lhs_mat_23_40_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp1, lhs_mat_23_41_sp1)); |
| 4834 | |
| 4835 | __m512i iacc_mat_00_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_01_51_sp1)); |
| 4836 | __m512i iacc_mat_01_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_01_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_01_51_sp1)); |
| 4837 | |
| 4838 | __m512i iacc_mat_10_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp1, lhs_mat_23_51_sp1)); |
| 4839 | __m512i iacc_mat_11_5_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp1, lhs_mat_23_50_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp1, lhs_mat_23_51_sp1)); |
| 4840 | |
| 4841 | __m512i iacc_mat_00_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_01_61_sp1)); |
| 4842 | __m512i iacc_mat_01_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_01_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_01_61_sp1)); |
| 4843 | |
| 4844 | __m512i iacc_mat_10_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp1, lhs_mat_23_61_sp1)); |
| 4845 | __m512i iacc_mat_11_6_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp1, lhs_mat_23_60_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp1, lhs_mat_23_61_sp1)); |
| 4846 | |
| 4847 | __m512i iacc_mat_00_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_01_71_sp1)); |
| 4848 | __m512i iacc_mat_01_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_01_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_01_71_sp1)); |
| 4849 | |
| 4850 | __m512i iacc_mat_10_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp1, lhs_mat_23_71_sp1)); |
| 4851 | __m512i iacc_mat_11_7_sp1 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp1, lhs_mat_23_70_sp1),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp1, lhs_mat_23_71_sp1)); |
| 4852 | |
| 4853 | |
| 4854 | __m512i iacc_mat_00_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_01_01_sp2)); |
| 4855 | __m512i iacc_mat_01_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_01_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_01_01_sp2)); |
| 4856 | |
| 4857 | __m512i iacc_mat_10_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_01_sp2, lhs_mat_23_01_sp2)); |
| 4858 | __m512i iacc_mat_11_0_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_00_sp2, lhs_mat_23_00_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_01_sp2, lhs_mat_23_01_sp2)); |
| 4859 | |
| 4860 | __m512i iacc_mat_00_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_01_11_sp2)); |
| 4861 | __m512i iacc_mat_01_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_01_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_01_11_sp2)); |
| 4862 | |
| 4863 | __m512i iacc_mat_10_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_11_sp2, lhs_mat_23_11_sp2)); |
| 4864 | __m512i iacc_mat_11_1_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_10_sp2, lhs_mat_23_10_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_11_sp2, lhs_mat_23_11_sp2)); |
| 4865 | |
| 4866 | __m512i iacc_mat_00_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_01_21_sp2)); |
| 4867 | __m512i iacc_mat_01_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_01_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_01_21_sp2)); |
| 4868 | |
| 4869 | __m512i iacc_mat_10_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_21_sp2, lhs_mat_23_21_sp2)); |
| 4870 | __m512i iacc_mat_11_2_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_20_sp2, lhs_mat_23_20_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_21_sp2, lhs_mat_23_21_sp2)); |
| 4871 | |
| 4872 | __m512i iacc_mat_00_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_01_31_sp2)); |
| 4873 | __m512i iacc_mat_01_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_01_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_01_31_sp2)); |
| 4874 | |
| 4875 | __m512i iacc_mat_10_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_31_sp2, lhs_mat_23_31_sp2)); |
| 4876 | __m512i iacc_mat_11_3_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_30_sp2, lhs_mat_23_30_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_31_sp2, lhs_mat_23_31_sp2)); |
| 4877 | |
| 4878 | __m512i iacc_mat_00_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_01_41_sp2)); |
| 4879 | __m512i iacc_mat_01_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_01_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_01_41_sp2)); |
| 4880 | |
| 4881 | __m512i iacc_mat_10_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_41_sp2, lhs_mat_23_41_sp2)); |
| 4882 | __m512i iacc_mat_11_4_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_40_sp2, lhs_mat_23_40_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_41_sp2, lhs_mat_23_41_sp2)); |
| 4883 | |
| 4884 | __m512i iacc_mat_00_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_01_51_sp2)); |
| 4885 | __m512i iacc_mat_01_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_01_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_01_51_sp2)); |
| 4886 | |
| 4887 | __m512i iacc_mat_10_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_51_sp2, lhs_mat_23_51_sp2)); |
| 4888 | __m512i iacc_mat_11_5_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_50_sp2, lhs_mat_23_50_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_51_sp2, lhs_mat_23_51_sp2)); |
| 4889 | |
| 4890 | __m512i iacc_mat_00_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_01_61_sp2)); |
| 4891 | __m512i iacc_mat_01_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_01_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_01_61_sp2)); |
| 4892 | |
| 4893 | __m512i iacc_mat_10_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_61_sp2, lhs_mat_23_61_sp2)); |
| 4894 | __m512i iacc_mat_11_6_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_60_sp2, lhs_mat_23_60_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_61_sp2, lhs_mat_23_61_sp2)); |
| 4895 | |
| 4896 | __m512i iacc_mat_00_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_01_71_sp2)); |
| 4897 | __m512i iacc_mat_01_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_01_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_01_71_sp2)); |
| 4898 | |
| 4899 | __m512i iacc_mat_10_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_014589CD_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_014589CD_71_sp2, lhs_mat_23_71_sp2)); |
| 4900 | __m512i iacc_mat_11_7_sp2 = _mm512_add_epi16(_mm512_maddubs_epi16(rhs_mat_2367ABEF_70_sp2, lhs_mat_23_70_sp2),_mm512_maddubs_epi16(rhs_mat_2367ABEF_71_sp2, lhs_mat_23_71_sp2)); |
| 4901 | |
| 4902 | // Combine results from both shuffle patterns for each output block |
| 4903 | __m512i iacc_mat_00_0 = _mm512_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 4904 | __m512i iacc_mat_01_0 = _mm512_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 4905 | __m512i iacc_mat_10_0 = _mm512_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 4906 | __m512i iacc_mat_11_0 = _mm512_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 4907 | |
| 4908 | __m512i iacc_mat_00_1 = _mm512_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 4909 | __m512i iacc_mat_01_1 = _mm512_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 4910 | __m512i iacc_mat_10_1 = _mm512_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 4911 | __m512i iacc_mat_11_1 = _mm512_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 4912 | |
| 4913 | __m512i iacc_mat_00_2 = _mm512_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2); |
| 4914 | __m512i iacc_mat_01_2 = _mm512_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2); |
| 4915 | __m512i iacc_mat_10_2 = _mm512_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2); |
| 4916 | __m512i iacc_mat_11_2 = _mm512_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2); |
| 4917 | |
| 4918 | __m512i iacc_mat_00_3 = _mm512_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2); |
| 4919 | __m512i iacc_mat_01_3 = _mm512_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2); |
| 4920 | __m512i iacc_mat_10_3 = _mm512_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2); |
| 4921 | __m512i iacc_mat_11_3 = _mm512_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2); |
| 4922 | |
| 4923 | __m512i iacc_mat_00_4 = _mm512_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2); |
| 4924 | __m512i iacc_mat_01_4 = _mm512_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2); |
| 4925 | __m512i iacc_mat_10_4 = _mm512_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2); |
| 4926 | __m512i iacc_mat_11_4 = _mm512_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2); |
| 4927 | |
| 4928 | __m512i iacc_mat_00_5 = _mm512_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2); |
| 4929 | __m512i iacc_mat_01_5 = _mm512_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2); |
| 4930 | __m512i iacc_mat_10_5 = _mm512_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2); |
| 4931 | __m512i iacc_mat_11_5 = _mm512_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2); |
| 4932 | |
| 4933 | __m512i iacc_mat_00_6 = _mm512_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2); |
| 4934 | __m512i iacc_mat_01_6 = _mm512_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2); |
| 4935 | __m512i iacc_mat_10_6 = _mm512_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2); |
| 4936 | __m512i iacc_mat_11_6 = _mm512_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2); |
| 4937 | |
| 4938 | __m512i iacc_mat_00_7 = _mm512_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2); |
| 4939 | __m512i iacc_mat_01_7 = _mm512_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2); |
| 4940 | __m512i iacc_mat_10_7 = _mm512_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2); |
| 4941 | __m512i iacc_mat_11_7 = _mm512_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2); |
| 4942 | |
| 4943 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 4944 | iacc_mat_00_0 = _mm512_madd_epi16(iacc_mat_00_0, scale_014589CD_0); |
| 4945 | iacc_mat_01_0 = _mm512_madd_epi16(iacc_mat_01_0, scale_2367ABEF_0); |
| 4946 | iacc_mat_10_0 = _mm512_madd_epi16(iacc_mat_10_0, scale_014589CD_0); |
| 4947 | iacc_mat_11_0 = _mm512_madd_epi16(iacc_mat_11_0, scale_2367ABEF_0); |
| 4948 | |
| 4949 | iacc_mat_00_1 = _mm512_madd_epi16(iacc_mat_00_1, scale_014589CD_1); |
| 4950 | iacc_mat_01_1 = _mm512_madd_epi16(iacc_mat_01_1, scale_2367ABEF_1); |
| 4951 | iacc_mat_10_1 = _mm512_madd_epi16(iacc_mat_10_1, scale_014589CD_1); |
| 4952 | iacc_mat_11_1 = _mm512_madd_epi16(iacc_mat_11_1, scale_2367ABEF_1); |
| 4953 | |
| 4954 | iacc_mat_00_2 = _mm512_madd_epi16(iacc_mat_00_2, scale_014589CD_2); |
| 4955 | iacc_mat_01_2 = _mm512_madd_epi16(iacc_mat_01_2, scale_2367ABEF_2); |
| 4956 | iacc_mat_10_2 = _mm512_madd_epi16(iacc_mat_10_2, scale_014589CD_2); |
| 4957 | iacc_mat_11_2 = _mm512_madd_epi16(iacc_mat_11_2, scale_2367ABEF_2); |
| 4958 | |
| 4959 | iacc_mat_00_3 = _mm512_madd_epi16(iacc_mat_00_3, scale_014589CD_3); |
| 4960 | iacc_mat_01_3 = _mm512_madd_epi16(iacc_mat_01_3, scale_2367ABEF_3); |
| 4961 | iacc_mat_10_3 = _mm512_madd_epi16(iacc_mat_10_3, scale_014589CD_3); |
| 4962 | iacc_mat_11_3 = _mm512_madd_epi16(iacc_mat_11_3, scale_2367ABEF_3); |
| 4963 | |
| 4964 | iacc_mat_00_4 = _mm512_madd_epi16(iacc_mat_00_4, scale_014589CD_4); |
| 4965 | iacc_mat_01_4 = _mm512_madd_epi16(iacc_mat_01_4, scale_2367ABEF_4); |
| 4966 | iacc_mat_10_4 = _mm512_madd_epi16(iacc_mat_10_4, scale_014589CD_4); |
| 4967 | iacc_mat_11_4 = _mm512_madd_epi16(iacc_mat_11_4, scale_2367ABEF_4); |
| 4968 | |
| 4969 | iacc_mat_00_5 = _mm512_madd_epi16(iacc_mat_00_5, scale_014589CD_5); |
| 4970 | iacc_mat_01_5 = _mm512_madd_epi16(iacc_mat_01_5, scale_2367ABEF_5); |
| 4971 | iacc_mat_10_5 = _mm512_madd_epi16(iacc_mat_10_5, scale_014589CD_5); |
| 4972 | iacc_mat_11_5 = _mm512_madd_epi16(iacc_mat_11_5, scale_2367ABEF_5); |
| 4973 | |
| 4974 | iacc_mat_00_6 = _mm512_madd_epi16(iacc_mat_00_6, scale_014589CD_6); |
| 4975 | iacc_mat_01_6 = _mm512_madd_epi16(iacc_mat_01_6, scale_2367ABEF_6); |
| 4976 | iacc_mat_10_6 = _mm512_madd_epi16(iacc_mat_10_6, scale_014589CD_6); |
| 4977 | iacc_mat_11_6 = _mm512_madd_epi16(iacc_mat_11_6, scale_2367ABEF_6); |
| 4978 | |
| 4979 | iacc_mat_00_7 = _mm512_madd_epi16(iacc_mat_00_7, scale_014589CD_7); |
| 4980 | iacc_mat_01_7 = _mm512_madd_epi16(iacc_mat_01_7, scale_2367ABEF_7); |
| 4981 | iacc_mat_10_7 = _mm512_madd_epi16(iacc_mat_10_7, scale_014589CD_7); |
| 4982 | iacc_mat_11_7 = _mm512_madd_epi16(iacc_mat_11_7, scale_2367ABEF_7); |
| 4983 | |
| 4984 | __m512i iacc_mat_00 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm512_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm512_add_epi32(iacc_mat_00_6, iacc_mat_00_7))); |
| 4985 | __m512i iacc_mat_01 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm512_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm512_add_epi32(iacc_mat_01_6, iacc_mat_01_7))); |
| 4986 | __m512i iacc_mat_10 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm512_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm512_add_epi32(iacc_mat_10_6, iacc_mat_10_7))); |
| 4987 | __m512i iacc_mat_11 = _mm512_add_epi32(_mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm512_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm512_add_epi32(_mm512_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm512_add_epi32(iacc_mat_11_6, iacc_mat_11_7))); |
| 4988 | |
| 4989 | // Straighten out to make 4 row vectors |
| 4990 | __m512i iacc_row_0 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_00, _mm512_shuffle_epi32(iacc_mat_01, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_01 ), (int)((_MM_PERM_ENUM)78)))); |
| 4991 | __m512i iacc_row_1 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_00, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_00 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_01); |
| 4992 | __m512i iacc_row_2 = _mm512_mask_blend_epi32(0xCCCC, iacc_mat_10, _mm512_shuffle_epi32(iacc_mat_11, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_11 ), (int)((_MM_PERM_ENUM)78)))); |
| 4993 | __m512i iacc_row_3 = _mm512_mask_blend_epi32(0xCCCC, _mm512_shuffle_epi32(iacc_mat_10, (_MM_PERM_ENUM)78)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(iacc_mat_10 ), (int)((_MM_PERM_ENUM)78))), iacc_mat_11); |
| 4994 | |
| 4995 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 4996 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); |
| 4997 | const __m256 row_scale_f32_ymm = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 4998 | const __m512 row_scale_f32 = _mm512_insertf32x8(_mm512_castps256_ps512(row_scale_f32_ymm), row_scale_f32_ymm, 1)((__m512)__builtin_ia32_insertf32x8((__v16sf)(__m512)(_mm512_castps256_ps512 (row_scale_f32_ymm)), (__v8sf)(__m256)(row_scale_f32_ymm), (int )(1))); |
| 4999 | |
| 5000 | // Multiply with appropiate scales and accumulate (for both d and dmin) below |
| 5001 | acc_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_0), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 5002 | acc_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_1), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 5003 | acc_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_2), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 5004 | acc_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_3), _mm512_mul_ps(col_scale_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 5005 | |
| 5006 | // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K |
| 5007 | __m512i iacc_row_min_0_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 5008 | __m512i iacc_row_min_1_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 5009 | __m512i iacc_row_min_2_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)0))), mins_01); |
| 5010 | __m512i iacc_row_min_3_01 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)170))), mins_01); |
| 5011 | |
| 5012 | __m512i iacc_row_min_0_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)85))), mins_23); |
| 5013 | __m512i iacc_row_min_1_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_0123, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_0123 ), (int)((_MM_PERM_ENUM)255))), mins_23); |
| 5014 | __m512i iacc_row_min_2_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)85))), mins_23); |
| 5015 | __m512i iacc_row_min_3_23 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_0123, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_0123 ), (int)((_MM_PERM_ENUM)255))), mins_23); |
| 5016 | |
| 5017 | __m512i iacc_row_min_0_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)0))), mins_45); |
| 5018 | __m512i iacc_row_min_1_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)170))), mins_45); |
| 5019 | __m512i iacc_row_min_2_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)0)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)0))), mins_45); |
| 5020 | __m512i iacc_row_min_3_45 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)170)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)170))), mins_45); |
| 5021 | |
| 5022 | __m512i iacc_row_min_0_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)85))), mins_67); |
| 5023 | __m512i iacc_row_min_1_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_01_4567, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_01_4567 ), (int)((_MM_PERM_ENUM)255))), mins_67); |
| 5024 | __m512i iacc_row_min_2_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)85)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)85))), mins_67); |
| 5025 | __m512i iacc_row_min_3_67 = _mm512_madd_epi16(_mm512_shuffle_epi32(lhs_bsums_23_4567, (_MM_PERM_ENUM)255)((__m512i)__builtin_ia32_pshufd512((__v16si)(__m512i)(lhs_bsums_23_4567 ), (int)((_MM_PERM_ENUM)255))), mins_67); |
| 5026 | |
| 5027 | __m512i iacc_row_min_0 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm512_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67)); |
| 5028 | __m512i iacc_row_min_1 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm512_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67)); |
| 5029 | __m512i iacc_row_min_2 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm512_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67)); |
| 5030 | __m512i iacc_row_min_3 = _mm512_add_epi32(_mm512_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm512_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67)); |
| 5031 | |
| 5032 | acc_min_rows[0] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_0), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(0)))), acc_min_rows[0]); |
| 5033 | acc_min_rows[1] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_1), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(85)))), acc_min_rows[1]); |
| 5034 | acc_min_rows[2] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_2), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(170)))), acc_min_rows[2]); |
| 5035 | acc_min_rows[3] = _mm512_fmadd_ps(_mm512_cvtepi32_ps(iacc_row_min_3), _mm512_mul_ps(col_dmin_f32, _mm512_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m512)__builtin_ia32_shufps512((__v16sf)(__m512)(row_scale_f32 ), (__v16sf)(__m512)(row_scale_f32), (int)(255)))), acc_min_rows[3]); |
| 5036 | } |
| 5037 | } |
| 5038 | // Store accumulated values |
| 5039 | for (int i = 0; i < 4; i++) { |
| 5040 | _mm512_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm512_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 5041 | } |
| 5042 | } |
| 5043 | } |
| 5044 | |
| 5045 | if (anc != nc) { |
| 5046 | xstart = anc/8; |
| 5047 | y = 0; |
| 5048 | } |
| 5049 | |
| 5050 | #endif // __AVX512BW__ && __AVX512DQ__ |
| 5051 | |
| 5052 | // Take group of four block_q8_Kx4 structures at each pass of the loop and perform dot product operation |
| 5053 | for (; y < anr / 4; y += 4) { |
| 5054 | |
| 5055 | const block_q8_Kx4 * a_ptrs[4]; |
| 5056 | |
| 5057 | a_ptrs[0] = a_ptr_start + (y * nb); |
| 5058 | for (int i = 0; i < 3; ++i) { |
| 5059 | a_ptrs[i + 1] = a_ptrs[i] + nb; |
| 5060 | } |
| 5061 | |
| 5062 | // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation |
| 5063 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 5064 | |
| 5065 | const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 5066 | |
| 5067 | // Master FP accumulators |
| 5068 | __m256 acc_rows[16]; |
| 5069 | for (int i = 0; i < 16; i++) { |
| 5070 | acc_rows[i] = _mm256_setzero_ps(); |
| 5071 | } |
| 5072 | |
| 5073 | __m256 acc_min_rows[16]; |
| 5074 | for (int i = 0; i < 16; i++) { |
| 5075 | acc_min_rows[i] = _mm256_setzero_ps(); |
| 5076 | } |
| 5077 | |
| 5078 | // For super block |
| 5079 | for (int64_t b = 0; b < nb; b++) { |
| 5080 | // Delta values - Load the eight scale values of block_q2_kx8 |
| 5081 | const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 5082 | |
| 5083 | // dmin values - Load the eight dmin values of block_q2_kx8 |
| 5084 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 5085 | |
| 5086 | // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration |
| 5087 | for (int sb = 0; sb < QK_K256 / 128; sb++) { |
| 5088 | |
| 5089 | // Load the eight block_q2_K for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 5090 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256)); |
| 5091 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256)); |
| 5092 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256)); |
| 5093 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256)); |
| 5094 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256)); |
| 5095 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256)); |
| 5096 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256)); |
| 5097 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256)); |
| 5098 | |
| 5099 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 5100 | //superblock sub block which part of sub block |
| 5101 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 5102 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 5103 | |
| 5104 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 5105 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 5106 | |
| 5107 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 5108 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 5109 | |
| 5110 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 5111 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 5112 | |
| 5113 | // 2-bit -> 8-bit |
| 5114 | // First sub block of the eight sub blocks processed in the iteration |
| 5115 | const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) |
| 5116 | const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) |
| 5117 | |
| 5118 | const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) |
| 5119 | const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) |
| 5120 | |
| 5121 | // Second sub block of the eight sub blocks processed in the iteration |
| 5122 | const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) |
| 5123 | const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) |
| 5124 | |
| 5125 | const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) |
| 5126 | const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) |
| 5127 | |
| 5128 | // Third sub block of the eight sub blocks processed in the iteration |
| 5129 | const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) |
| 5130 | const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) |
| 5131 | |
| 5132 | const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) |
| 5133 | const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) |
| 5134 | |
| 5135 | // Fourth sub block of the eight sub blocks processed in the iteration |
| 5136 | const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) |
| 5137 | const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) |
| 5138 | |
| 5139 | const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) |
| 5140 | const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) |
| 5141 | |
| 5142 | // Fifth sub block of the eight sub blocks processed in the iteration |
| 5143 | const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) |
| 5144 | const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) |
| 5145 | |
| 5146 | const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) |
| 5147 | const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) |
| 5148 | |
| 5149 | // Sixth sub block of the eight sub blocks processed in the iteration |
| 5150 | const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) |
| 5151 | const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) |
| 5152 | |
| 5153 | const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) |
| 5154 | const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) |
| 5155 | |
| 5156 | // Seventh sub block of the eight sub blocks processed in the iteration |
| 5157 | const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) |
| 5158 | const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) |
| 5159 | |
| 5160 | const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) |
| 5161 | const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) |
| 5162 | |
| 5163 | // Eighth sub block of the eight sub blocks processed in the iteration |
| 5164 | const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) |
| 5165 | const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) |
| 5166 | |
| 5167 | const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) |
| 5168 | const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) |
| 5169 | |
| 5170 | // Shuffle pattern one - right side input |
| 5171 | const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) |
| 5172 | const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) |
| 5173 | |
| 5174 | const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) |
| 5175 | const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) |
| 5176 | |
| 5177 | const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) |
| 5178 | const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) |
| 5179 | |
| 5180 | const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) |
| 5181 | const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) |
| 5182 | |
| 5183 | const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_20 ), (int)(136))); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) |
| 5184 | const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_20 ), (int)(136))); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) |
| 5185 | |
| 5186 | const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_21 ), (int)(136))); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) |
| 5187 | const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_21 ), (int)(136))); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) |
| 5188 | |
| 5189 | const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_30 ), (int)(136))); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) |
| 5190 | const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_30 ), (int)(136))); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) |
| 5191 | |
| 5192 | const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_31 ), (int)(136))); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11 |
| 5193 | const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_31 ), (int)(136))); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) |
| 5194 | |
| 5195 | const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_40 ), (int)(136))); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) |
| 5196 | const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_40 ), (int)(136))); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) |
| 5197 | |
| 5198 | const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_41 ), (int)(136))); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) |
| 5199 | const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_41 ), (int)(136))); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) |
| 5200 | |
| 5201 | const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_50 ), (int)(136))); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) |
| 5202 | const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_50 ), (int)(136))); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) |
| 5203 | |
| 5204 | const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_51 ), (int)(136))); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) |
| 5205 | const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_51 ), (int)(136))); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) |
| 5206 | |
| 5207 | const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_60 ), (int)(136))); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) |
| 5208 | const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_60 ), (int)(136))); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) |
| 5209 | |
| 5210 | const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_61 ), (int)(136))); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) |
| 5211 | const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_61 ), (int)(136))); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) |
| 5212 | |
| 5213 | const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_70 ), (int)(136))); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) |
| 5214 | const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_70 ), (int)(136))); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) |
| 5215 | |
| 5216 | const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_71 ), (int)(136))); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11) |
| 5217 | const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_71 ), (int)(136))); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) |
| 5218 | |
| 5219 | |
| 5220 | // Shuffle pattern two - right side input |
| 5221 | const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) |
| 5222 | const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) |
| 5223 | |
| 5224 | const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) |
| 5225 | const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) |
| 5226 | |
| 5227 | const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) |
| 5228 | const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) |
| 5229 | |
| 5230 | const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) |
| 5231 | const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) |
| 5232 | |
| 5233 | const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_20 ), (int)(221))); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) |
| 5234 | const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_20 ), (int)(221))); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) |
| 5235 | |
| 5236 | const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_21 ), (int)(221))); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) |
| 5237 | const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_21 ), (int)(221))); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) |
| 5238 | |
| 5239 | const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_30 ), (int)(221))); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) |
| 5240 | const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_30 ), (int)(221))); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) |
| 5241 | |
| 5242 | const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_31 ), (int)(221))); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) |
| 5243 | const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_31 ), (int)(221))); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) |
| 5244 | |
| 5245 | const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_40 ), (int)(221))); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) |
| 5246 | const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_40 ), (int)(221))); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) |
| 5247 | |
| 5248 | const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_41 ), (int)(221))); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) |
| 5249 | const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_41 ), (int)(221))); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) |
| 5250 | |
| 5251 | const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_50 ), (int)(221))); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) |
| 5252 | const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_50 ), (int)(221))); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) |
| 5253 | |
| 5254 | const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_51 ), (int)(221))); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) |
| 5255 | const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_51 ), (int)(221))); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) |
| 5256 | |
| 5257 | const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_60 ), (int)(221))); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) |
| 5258 | const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_60 ), (int)(221))); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) |
| 5259 | |
| 5260 | const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_61 ), (int)(221))); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) |
| 5261 | const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_61 ), (int)(221))); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) |
| 5262 | |
| 5263 | const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_70 ), (int)(221))); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) |
| 5264 | const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_70 ), (int)(221))); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) |
| 5265 | |
| 5266 | const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_71 ), (int)(221))); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) |
| 5267 | const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_71 ), (int)(221))); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) |
| 5268 | |
| 5269 | //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together |
| 5270 | //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71 |
| 5271 | |
| 5272 | // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop |
| 5273 | const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64)); |
| 5274 | const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64)); |
| 5275 | const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64)); |
| 5276 | const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64)); |
| 5277 | |
| 5278 | // Extract scales which is lower half from mins_and_scales |
| 5279 | const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse); |
| 5280 | const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse); |
| 5281 | const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse); |
| 5282 | const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse); |
| 5283 | |
| 5284 | // Extract mins which is upper half from mins_and_scales |
| 5285 | const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse)); |
| 5286 | const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse)); |
| 5287 | const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse)); |
| 5288 | const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse)); |
| 5289 | |
| 5290 | const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse)); |
| 5291 | const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse)); |
| 5292 | |
| 5293 | const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse)); |
| 5294 | const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse)); |
| 5295 | |
| 5296 | const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse)); |
| 5297 | const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse)); |
| 5298 | |
| 5299 | const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse)); |
| 5300 | const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse)); |
| 5301 | |
| 5302 | const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(68))); |
| 5303 | const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(238))); |
| 5304 | |
| 5305 | const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(68))); |
| 5306 | const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(238))); |
| 5307 | |
| 5308 | const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_2 ), (int)(68))); |
| 5309 | const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_2 ), (int)(238))); |
| 5310 | |
| 5311 | const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_3 ), (int)(68))); |
| 5312 | const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_3 ), (int)(238))); |
| 5313 | |
| 5314 | const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_4 ), (int)(68))); |
| 5315 | const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_4 ), (int)(238))); |
| 5316 | |
| 5317 | const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_5 ), (int)(68))); |
| 5318 | const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_5 ), (int)(238))); |
| 5319 | |
| 5320 | const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_6 ), (int)(68))); |
| 5321 | const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_6 ), (int)(238))); |
| 5322 | |
| 5323 | const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_7 ), (int)(68))); |
| 5324 | const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_7 ), (int)(238))); |
| 5325 | |
| 5326 | |
| 5327 | for (int rp = 0; rp < 4; rp++) { |
| 5328 | |
| 5329 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 5330 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 5331 | __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 512 * sb))); |
| 5332 | __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(0))); |
| 5333 | __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(17))); |
| 5334 | __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 32 + 512 * sb))); |
| 5335 | __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(0))); |
| 5336 | __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(17))); |
| 5337 | __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 64 + 512 * sb))); |
| 5338 | __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(0))); |
| 5339 | __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(17))); |
| 5340 | __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 96 + 512 * sb))); |
| 5341 | __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(0))); |
| 5342 | __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(17))); |
| 5343 | __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 128 + 512 * sb))); |
| 5344 | __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_20 ), (__v8si)(__m256i)(lhs_mat_0123_20), (int)(0))); |
| 5345 | __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_20 ), (__v8si)(__m256i)(lhs_mat_0123_20), (int)(17))); |
| 5346 | __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 160 + 512 * sb))); |
| 5347 | __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_21 ), (__v8si)(__m256i)(lhs_mat_0123_21), (int)(0))); |
| 5348 | __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_21 ), (__v8si)(__m256i)(lhs_mat_0123_21), (int)(17))); |
| 5349 | __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 192 + 512 * sb))); |
| 5350 | __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_30 ), (__v8si)(__m256i)(lhs_mat_0123_30), (int)(0))); |
| 5351 | __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_30 ), (__v8si)(__m256i)(lhs_mat_0123_30), (int)(17))); |
| 5352 | __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 224 + 512 * sb))); |
| 5353 | __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_31 ), (__v8si)(__m256i)(lhs_mat_0123_31), (int)(0))); |
| 5354 | __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_31 ), (__v8si)(__m256i)(lhs_mat_0123_31), (int)(17))); |
| 5355 | |
| 5356 | __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 256 + 512 * sb))); |
| 5357 | __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_40 ), (__v8si)(__m256i)(lhs_mat_0123_40), (int)(0))); |
| 5358 | __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_40 ), (__v8si)(__m256i)(lhs_mat_0123_40), (int)(17))); |
| 5359 | __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 288 + 512 * sb))); |
| 5360 | __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_41 ), (__v8si)(__m256i)(lhs_mat_0123_41), (int)(0))); |
| 5361 | __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_41 ), (__v8si)(__m256i)(lhs_mat_0123_41), (int)(17))); |
| 5362 | __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 320 + 512 * sb))); |
| 5363 | __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_50 ), (__v8si)(__m256i)(lhs_mat_0123_50), (int)(0))); |
| 5364 | __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_50 ), (__v8si)(__m256i)(lhs_mat_0123_50), (int)(17))); |
| 5365 | __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 352 + 512 * sb))); |
| 5366 | __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_51 ), (__v8si)(__m256i)(lhs_mat_0123_51), (int)(0))); |
| 5367 | __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_51 ), (__v8si)(__m256i)(lhs_mat_0123_51), (int)(17))); |
| 5368 | __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 384 + 512 * sb))); |
| 5369 | __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_60 ), (__v8si)(__m256i)(lhs_mat_0123_60), (int)(0))); |
| 5370 | __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_60 ), (__v8si)(__m256i)(lhs_mat_0123_60), (int)(17))); |
| 5371 | __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 416 + 512 * sb))); |
| 5372 | __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_61 ), (__v8si)(__m256i)(lhs_mat_0123_61), (int)(0))); |
| 5373 | __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_61 ), (__v8si)(__m256i)(lhs_mat_0123_61), (int)(17))); |
| 5374 | __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 448 + 512 * sb))); |
| 5375 | __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_70 ), (__v8si)(__m256i)(lhs_mat_0123_70), (int)(0))); |
| 5376 | __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_70 ), (__v8si)(__m256i)(lhs_mat_0123_70), (int)(17))); |
| 5377 | __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptrs[rp][b].qs + 480 + 512 * sb))); |
| 5378 | __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_71 ), (__v8si)(__m256i)(lhs_mat_0123_71), (int)(0))); |
| 5379 | __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_71 ), (__v8si)(__m256i)(lhs_mat_0123_71), (int)(17))); |
| 5380 | |
| 5381 | // Bsums are loaded for the different Q8_K blocks |
| 5382 | __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 32 * sb))); |
| 5383 | __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 8 + 32 * sb)); |
| 5384 | __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptrs[rp][b].bsums + 16 + 32 * sb))); |
| 5385 | __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptrs[rp][b].bsums + 24 + 32 * sb)); |
| 5386 | |
| 5387 | // Shuffle pattern one - left side input |
| 5388 | const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 5389 | const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(160))); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) |
| 5390 | |
| 5391 | const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 5392 | const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(160))); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) |
| 5393 | |
| 5394 | const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 5395 | const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(160))); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) |
| 5396 | |
| 5397 | const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 5398 | const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(160))); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) |
| 5399 | |
| 5400 | const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_20 ), (int)(160))); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) |
| 5401 | const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_20 ), (int)(160))); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) |
| 5402 | |
| 5403 | const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_21 ), (int)(160))); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) |
| 5404 | const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_21 ), (int)(160))); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) |
| 5405 | |
| 5406 | const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_30 ), (int)(160))); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) |
| 5407 | const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_30 ), (int)(160))); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) |
| 5408 | |
| 5409 | const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_31 ), (int)(160))); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) |
| 5410 | const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_31 ), (int)(160))); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) |
| 5411 | |
| 5412 | const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_40 ), (int)(160))); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) |
| 5413 | const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_40 ), (int)(160))); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) |
| 5414 | |
| 5415 | const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_41 ), (int)(160))); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) |
| 5416 | const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_41 ), (int)(160))); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) |
| 5417 | |
| 5418 | const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_50 ), (int)(160))); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) |
| 5419 | const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_50 ), (int)(160))); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) |
| 5420 | |
| 5421 | const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_51 ), (int)(160))); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) |
| 5422 | const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_51 ), (int)(160))); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) |
| 5423 | |
| 5424 | const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_60 ), (int)(160))); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) |
| 5425 | const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_60 ), (int)(160))); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) |
| 5426 | |
| 5427 | const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_61 ), (int)(160))); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) |
| 5428 | const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_61 ), (int)(160))); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) |
| 5429 | |
| 5430 | const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_70 ), (int)(160))); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) |
| 5431 | const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_70 ), (int)(160))); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) |
| 5432 | |
| 5433 | const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_71 ), (int)(160))); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) |
| 5434 | const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_71 ), (int)(160))); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) |
| 5435 | |
| 5436 | // Shuffle pattern two- left side input |
| 5437 | const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 5438 | const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(245))); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) |
| 5439 | |
| 5440 | const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 5441 | const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(245))); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) |
| 5442 | |
| 5443 | const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 5444 | const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(245))); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) |
| 5445 | |
| 5446 | const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 5447 | const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(245))); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) |
| 5448 | |
| 5449 | const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_20 ), (int)(245))); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) |
| 5450 | const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_20 ), (int)(245))); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) |
| 5451 | |
| 5452 | const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_21 ), (int)(245))); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) |
| 5453 | const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_21 ), (int)(245))); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) |
| 5454 | |
| 5455 | const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_30 ), (int)(245))); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) |
| 5456 | const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_30 ), (int)(245))); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) |
| 5457 | |
| 5458 | const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_31 ), (int)(245))); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) |
| 5459 | const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_31 ), (int)(245))); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) |
| 5460 | |
| 5461 | const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_40 ), (int)(245))); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) |
| 5462 | const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_40 ), (int)(245))); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) |
| 5463 | |
| 5464 | const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_41 ), (int)(245))); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) |
| 5465 | const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_41 ), (int)(245))); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) |
| 5466 | |
| 5467 | const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_50 ), (int)(245))); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) |
| 5468 | const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_50 ), (int)(245))); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) |
| 5469 | |
| 5470 | const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_51 ), (int)(245))); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) |
| 5471 | const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_51 ), (int)(245))); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) |
| 5472 | |
| 5473 | const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_60 ), (int)(245))); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) |
| 5474 | const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_60 ), (int)(245))); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) |
| 5475 | |
| 5476 | const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_61 ), (int)(245))); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) |
| 5477 | const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_61 ), (int)(245))); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) |
| 5478 | |
| 5479 | const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_70 ), (int)(245))); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) |
| 5480 | const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_70 ), (int)(245))); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) |
| 5481 | |
| 5482 | const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_71 ), (int)(245))); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) |
| 5483 | const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_71 ), (int)(245))); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) |
| 5484 | |
| 5485 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 5486 | __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)); |
| 5487 | __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)); |
| 5488 | |
| 5489 | __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)); |
| 5490 | __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)); |
| 5491 | |
| 5492 | __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)); |
| 5493 | __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)); |
| 5494 | |
| 5495 | __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)); |
| 5496 | __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)); |
| 5497 | |
| 5498 | __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1)); |
| 5499 | __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1)); |
| 5500 | |
| 5501 | __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1)); |
| 5502 | __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1)); |
| 5503 | |
| 5504 | __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1)); |
| 5505 | __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1)); |
| 5506 | |
| 5507 | __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1)); |
| 5508 | __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1)); |
| 5509 | |
| 5510 | __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1)); |
| 5511 | __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1)); |
| 5512 | |
| 5513 | __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1)); |
| 5514 | __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1)); |
| 5515 | |
| 5516 | __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1)); |
| 5517 | __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1)); |
| 5518 | |
| 5519 | __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1)); |
| 5520 | __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1)); |
| 5521 | |
| 5522 | __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1)); |
| 5523 | __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1)); |
| 5524 | |
| 5525 | __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1)); |
| 5526 | __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1)); |
| 5527 | |
| 5528 | __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1)); |
| 5529 | __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1)); |
| 5530 | |
| 5531 | __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1)); |
| 5532 | __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1)); |
| 5533 | |
| 5534 | |
| 5535 | __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)); |
| 5536 | __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)); |
| 5537 | |
| 5538 | __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)); |
| 5539 | __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)); |
| 5540 | |
| 5541 | __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)); |
| 5542 | __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)); |
| 5543 | |
| 5544 | __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)); |
| 5545 | __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)); |
| 5546 | |
| 5547 | __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2)); |
| 5548 | __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2)); |
| 5549 | |
| 5550 | __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2)); |
| 5551 | __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2)); |
| 5552 | |
| 5553 | __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2)); |
| 5554 | __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2)); |
| 5555 | |
| 5556 | __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2)); |
| 5557 | __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2)); |
| 5558 | |
| 5559 | __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2)); |
| 5560 | __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2)); |
| 5561 | |
| 5562 | __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2)); |
| 5563 | __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2)); |
| 5564 | |
| 5565 | __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2)); |
| 5566 | __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2)); |
| 5567 | |
| 5568 | __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2)); |
| 5569 | __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2)); |
| 5570 | |
| 5571 | __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2)); |
| 5572 | __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2)); |
| 5573 | |
| 5574 | __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2)); |
| 5575 | __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2)); |
| 5576 | |
| 5577 | __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2)); |
| 5578 | __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2)); |
| 5579 | |
| 5580 | __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2)); |
| 5581 | __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2)); |
| 5582 | |
| 5583 | // Combine results from both shuffle patterns for each output block |
| 5584 | __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 5585 | __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 5586 | __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 5587 | __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 5588 | |
| 5589 | __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 5590 | __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 5591 | __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 5592 | __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 5593 | |
| 5594 | __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2); |
| 5595 | __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2); |
| 5596 | __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2); |
| 5597 | __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2); |
| 5598 | |
| 5599 | __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2); |
| 5600 | __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2); |
| 5601 | __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2); |
| 5602 | __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2); |
| 5603 | |
| 5604 | __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2); |
| 5605 | __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2); |
| 5606 | __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2); |
| 5607 | __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2); |
| 5608 | |
| 5609 | __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2); |
| 5610 | __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2); |
| 5611 | __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2); |
| 5612 | __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2); |
| 5613 | |
| 5614 | __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2); |
| 5615 | __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2); |
| 5616 | __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2); |
| 5617 | __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2); |
| 5618 | |
| 5619 | __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2); |
| 5620 | __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2); |
| 5621 | __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2); |
| 5622 | __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2); |
| 5623 | |
| 5624 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 5625 | iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); |
| 5626 | iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); |
| 5627 | iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); |
| 5628 | iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); |
| 5629 | |
| 5630 | iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); |
| 5631 | iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); |
| 5632 | iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); |
| 5633 | iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); |
| 5634 | |
| 5635 | iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2); |
| 5636 | iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2); |
| 5637 | iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2); |
| 5638 | iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2); |
| 5639 | |
| 5640 | iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3); |
| 5641 | iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3); |
| 5642 | iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3); |
| 5643 | iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3); |
| 5644 | |
| 5645 | iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4); |
| 5646 | iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4); |
| 5647 | iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4); |
| 5648 | iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4); |
| 5649 | |
| 5650 | iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5); |
| 5651 | iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5); |
| 5652 | iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5); |
| 5653 | iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5); |
| 5654 | |
| 5655 | iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6); |
| 5656 | iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6); |
| 5657 | iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6); |
| 5658 | iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6); |
| 5659 | |
| 5660 | iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7); |
| 5661 | iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7); |
| 5662 | iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7); |
| 5663 | iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7); |
| 5664 | |
| 5665 | __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7))); |
| 5666 | __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7))); |
| 5667 | __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7))); |
| 5668 | __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7))); |
| 5669 | |
| 5670 | // Straighten out to make 4 row vectors |
| 5671 | __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01), (int)(78)))), (int)(204))); |
| 5672 | __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00), (int )(78)))), (__v8si)(__m256i)(iacc_mat_01), (int)(204))); |
| 5673 | __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11), (int)(78)))), (int)(204))); |
| 5674 | __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10), (int )(78)))), (__v8si)(__m256i)(iacc_mat_11), (int)(204))); |
| 5675 | |
| 5676 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 5677 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptrs[rp][b].d); |
| 5678 | const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 5679 | |
| 5680 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 5681 | acc_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[rp * 4]); |
| 5682 | acc_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[rp * 4 + 1]); |
| 5683 | acc_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[rp * 4 + 2]); |
| 5684 | acc_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[rp * 4 + 3]); |
| 5685 | |
| 5686 | __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_0123)), (__v2di)(__m128i)(lhs_raw_bsums_01_0123 ), (int)(1))); |
| 5687 | __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_0123)), (__v2di)(__m128i)(lhs_raw_bsums_23_0123 ), (int)(1))); |
| 5688 | __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_4567)), (__v2di)(__m128i)(lhs_raw_bsums_01_4567 ), (int)(1))); |
| 5689 | __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_4567)), (__v2di)(__m128i)(lhs_raw_bsums_23_4567 ), (int)(1))); |
| 5690 | |
| 5691 | // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K |
| 5692 | __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(0))), mins_01); |
| 5693 | __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(170))), mins_01); |
| 5694 | __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(0))), mins_01); |
| 5695 | __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(170))), mins_01); |
| 5696 | |
| 5697 | __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(85))), mins_23); |
| 5698 | __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(255))), mins_23); |
| 5699 | __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(85))), mins_23); |
| 5700 | __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(255))), mins_23); |
| 5701 | |
| 5702 | __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(0))), mins_45); |
| 5703 | __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(170))), mins_45); |
| 5704 | __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(0))), mins_45); |
| 5705 | __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(170))), mins_45); |
| 5706 | |
| 5707 | __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(85))), mins_67); |
| 5708 | __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(255))), mins_67); |
| 5709 | __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(85))), mins_67); |
| 5710 | __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(255))), mins_67); |
| 5711 | |
| 5712 | __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67)); |
| 5713 | __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67)); |
| 5714 | __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67)); |
| 5715 | __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67)); |
| 5716 | |
| 5717 | acc_min_rows[rp * 4] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_min_rows[rp * 4]); |
| 5718 | acc_min_rows[rp * 4 + 1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_min_rows[rp * 4 + 1]); |
| 5719 | acc_min_rows[rp * 4 + 2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_min_rows[rp * 4 + 2]); |
| 5720 | acc_min_rows[rp * 4 + 3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_min_rows[rp * 4 + 3]); |
| 5721 | |
| 5722 | } |
| 5723 | } |
| 5724 | } |
| 5725 | // Store the accumulated values |
| 5726 | for (int i = 0; i < 16; i++) { |
| 5727 | _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 5728 | |
| 5729 | } |
| 5730 | } |
| 5731 | } |
| 5732 | |
| 5733 | for (; y < nr / 4; y ++) { |
| 5734 | |
| 5735 | const block_q8_Kx4 * a_ptr = a_ptr_start + (y * nb); |
| 5736 | |
| 5737 | // Take group of eight block_q2_kx8 structures at each pass of the loop and perform dot product operation |
| 5738 | for (int64_t x = xstart; x < nc / 8; x++) { |
| 5739 | |
| 5740 | const block_q2_Kx8 * b_ptr = b_ptr_start + (x * b_nb); |
| 5741 | |
| 5742 | // Master FP accumulators |
| 5743 | __m256 acc_rows[4]; |
| 5744 | for (int i = 0; i < 4; i++) { |
| 5745 | acc_rows[i] = _mm256_setzero_ps(); |
| 5746 | } |
| 5747 | |
| 5748 | __m256 acc_min_rows[4]; |
| 5749 | for (int i = 0; i < 4; i++) { |
| 5750 | acc_min_rows[i] = _mm256_setzero_ps(); |
| 5751 | } |
| 5752 | |
| 5753 | for (int64_t b = 0; b < nb; b++) { |
| 5754 | // Delta values - Load the eight scale values of block_q2_kx8 |
| 5755 | const __m256 col_scale_f32 = GGML_F32Cx8_LOAD(b_ptr[b].d)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].d) )); |
| 5756 | |
| 5757 | // dmin values - Load the eight dmin values of block_q2_kx8 |
| 5758 | const __m256 col_dmin_f32 = GGML_F32Cx8_LOAD(b_ptr[b].dmin)_mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(b_ptr[b].dmin ))); |
| 5759 | |
| 5760 | // Loop to iterate over the sixteen sub blocks of a super block - eight sub blocks are processed per iteration |
| 5761 | for (int sb = 0; sb < QK_K256 / 128; sb++) { |
| 5762 | |
| 5763 | // Load the eight block_q2_k for eight sub blocks quantized values interleaved with each other in chunks of eight bytes - B0,B1 ....B6,B7 |
| 5764 | const __m256i rhs_raw_mat_0123_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + sb * 256)); |
| 5765 | const __m256i rhs_raw_mat_4567_0 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 32 + sb * 256)); |
| 5766 | const __m256i rhs_raw_mat_0123_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 64 + sb * 256)); |
| 5767 | const __m256i rhs_raw_mat_4567_1 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 96 + sb * 256)); |
| 5768 | const __m256i rhs_raw_mat_0123_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 128 + sb * 256)); |
| 5769 | const __m256i rhs_raw_mat_4567_2 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 160 + sb * 256)); |
| 5770 | const __m256i rhs_raw_mat_0123_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 192 + sb * 256)); |
| 5771 | const __m256i rhs_raw_mat_4567_3 = _mm256_loadu_si256((const __m256i *)(b_ptr[b].qs + 224 + sb * 256)); |
| 5772 | |
| 5773 | // Save the values in the following vectors in the formats B0B1B4B5, B2B3B6B7 for further processing and storing of values |
| 5774 | //superblock sub block which part of sub block |
| 5775 | const __m256i rhs_raw_mat_0145_0 = _mm256_blend_epi32(rhs_raw_mat_0123_0, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_0 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_0 , requiredOrder)), (int)(240))); |
| 5776 | const __m256i rhs_raw_mat_2367_0 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_0, requiredOrder), rhs_raw_mat_4567_0, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_0, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_0 ), (int)(240))); |
| 5777 | |
| 5778 | const __m256i rhs_raw_mat_0145_1 = _mm256_blend_epi32(rhs_raw_mat_0123_1, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_1 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_1 , requiredOrder)), (int)(240))); |
| 5779 | const __m256i rhs_raw_mat_2367_1 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_1, requiredOrder), rhs_raw_mat_4567_1, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_1, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_1 ), (int)(240))); |
| 5780 | |
| 5781 | const __m256i rhs_raw_mat_0145_2 = _mm256_blend_epi32(rhs_raw_mat_0123_2, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_2 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_2 , requiredOrder)), (int)(240))); |
| 5782 | const __m256i rhs_raw_mat_2367_2 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_2, requiredOrder), rhs_raw_mat_4567_2, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_2, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_2 ), (int)(240))); |
| 5783 | |
| 5784 | const __m256i rhs_raw_mat_0145_3 = _mm256_blend_epi32(rhs_raw_mat_0123_3, _mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3, requiredOrder), 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(rhs_raw_mat_0123_3 ), (__v8si)(__m256i)(_mm256_permutevar8x32_epi32(rhs_raw_mat_4567_3 , requiredOrder)), (int)(240))); |
| 5785 | const __m256i rhs_raw_mat_2367_3 = _mm256_blend_epi32(_mm256_permutevar8x32_epi32(rhs_raw_mat_0123_3, requiredOrder), rhs_raw_mat_4567_3, 240)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(_mm256_permutevar8x32_epi32 (rhs_raw_mat_0123_3, requiredOrder)), (__v8si)(__m256i)(rhs_raw_mat_4567_3 ), (int)(240))); |
| 5786 | |
| 5787 | // 2-bit -> 8-bit |
| 5788 | // First sub block of the eight sub blocks processed in the iteration |
| 5789 | const __m256i rhs_mat_0145_00 = _mm256_and_si256(rhs_raw_mat_0145_0, m3b); //B00(0-7) B01(0-7) B04(0-7) B05(0-7) |
| 5790 | const __m256i rhs_mat_2367_00 = _mm256_and_si256(rhs_raw_mat_2367_0, m3b); //B02(0-7) B03(0-7) B06(0-7) B07(0-7) |
| 5791 | |
| 5792 | const __m256i rhs_mat_0145_01 = _mm256_and_si256(rhs_raw_mat_0145_1, m3b); //B00(8-15) B01(8-15) B04(8-15) B05(8-15) |
| 5793 | const __m256i rhs_mat_2367_01 = _mm256_and_si256(rhs_raw_mat_2367_1, m3b); //B02(8-15) B03(8-15) B06(8-15) B07(8-15) |
| 5794 | |
| 5795 | // Second sub block of the eight sub blocks processed in the iteration |
| 5796 | const __m256i rhs_mat_0145_10 = _mm256_and_si256(rhs_raw_mat_0145_2, m3b); //B10(0-7) B11(0-7) B14(0-7) B15(0-7) |
| 5797 | const __m256i rhs_mat_2367_10 = _mm256_and_si256(rhs_raw_mat_2367_2, m3b); //B12(0-7) B13(0-7) B16(0-7) B17(0-7) |
| 5798 | |
| 5799 | const __m256i rhs_mat_0145_11 = _mm256_and_si256(rhs_raw_mat_0145_3, m3b); //B10(8-15) B11(8-15) B14(8-15) B15(8-15) |
| 5800 | const __m256i rhs_mat_2367_11 = _mm256_and_si256(rhs_raw_mat_2367_3, m3b); //B12(8-15) B13(8-15) B16(8-15) B17(8-15) |
| 5801 | |
| 5802 | // Third sub block of the eight sub blocks processed in the iteration |
| 5803 | const __m256i rhs_mat_0145_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 2), m3b); //B20(0-7) B21(0-7) B24(0-7) B25(0-7) |
| 5804 | const __m256i rhs_mat_2367_20 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 2), m3b); //B22(0-7) B23(0-7) B26(0-7) B27(0-7) |
| 5805 | |
| 5806 | const __m256i rhs_mat_0145_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 2), m3b); //B20(8-15) B21(8-15) B24(8-15) B25(8-15) |
| 5807 | const __m256i rhs_mat_2367_21 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 2), m3b); //B22(8-15) B23(8-15) B26(8-15) B27(8-15) |
| 5808 | |
| 5809 | // Fourth sub block of the eight sub blocks processed in the iteration |
| 5810 | const __m256i rhs_mat_0145_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 2), m3b); //B30(0-7) B31(0-7) B34(0-7) B35(0-7) |
| 5811 | const __m256i rhs_mat_2367_30 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 2), m3b); //B32(0-7) B33(0-7) B36(0-7) B37(0-7) |
| 5812 | |
| 5813 | const __m256i rhs_mat_0145_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 2), m3b); //B30(8-15) B31(8-15) B34(8-15) B35(8-15) |
| 5814 | const __m256i rhs_mat_2367_31 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 2), m3b); //B32(8-15) B33(8-15) B36(8-15) B37(8-15) |
| 5815 | |
| 5816 | // Fifth sub block of the eight sub blocks processed in the iteration |
| 5817 | const __m256i rhs_mat_0145_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 4), m3b); //B40(0-7) B41(0-7) B44(0-7) B45(0-7) |
| 5818 | const __m256i rhs_mat_2367_40 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 4), m3b); //B42(0-7) B43(0-7) B46(0-7) B47(0-7) |
| 5819 | |
| 5820 | const __m256i rhs_mat_0145_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 4), m3b); //B40(8-15) B41(8-15) B44(8-15) B45(8-15) |
| 5821 | const __m256i rhs_mat_2367_41 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 4), m3b); //B42(8-15) B43(8-15) B46(8-15) B47(8-15) |
| 5822 | |
| 5823 | // Sixth sub block of the eight sub blocks processed in the iteration |
| 5824 | const __m256i rhs_mat_0145_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 4), m3b); //B50(0-7) B51(0-7) B54(0-7) B55(0-7) |
| 5825 | const __m256i rhs_mat_2367_50 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 4), m3b); //B52(0-7) B53(0-7) B56(0-7) B57(0-7) |
| 5826 | |
| 5827 | const __m256i rhs_mat_0145_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 4), m3b); //B50(8-15) B51(8-15) B54(8-15) B55(8-15) |
| 5828 | const __m256i rhs_mat_2367_51 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 4), m3b); //B52(8-15) B53(8-15) B56(8-15) B57(8-15) |
| 5829 | |
| 5830 | // Seventh sub block of the eight sub blocks processed in the iteration |
| 5831 | const __m256i rhs_mat_0145_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_0, 6), m3b); //B60(0-7) B61(0-7) B64(0-7) B65(0-7) |
| 5832 | const __m256i rhs_mat_2367_60 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_0, 6), m3b); //B62(0-7) B63(0-7) B66(0-7) B67(0-7) |
| 5833 | |
| 5834 | const __m256i rhs_mat_0145_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_1, 6), m3b); //B60(8-15) B61(8-15) B64(8-15) B65(8-15) |
| 5835 | const __m256i rhs_mat_2367_61 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_1, 6), m3b); //B62(8-15) B63(8-15) B66(8-15) B67(8-15) |
| 5836 | |
| 5837 | // Eighth sub block of the eight sub blocks processed in the iteration |
| 5838 | const __m256i rhs_mat_0145_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_2, 6), m3b); //B70(0-7) B71(0-7) B74(0-7) B75(0-7) |
| 5839 | const __m256i rhs_mat_2367_70 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_2, 6), m3b); //B72(0-7) B73(0-7) B76(0-7) B77(0-7) |
| 5840 | |
| 5841 | const __m256i rhs_mat_0145_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_0145_3, 6), m3b); //B70(8-15) B71(8-15) B74(8-15) B75(8-15) |
| 5842 | const __m256i rhs_mat_2367_71 = _mm256_and_si256(_mm256_srli_epi16(rhs_raw_mat_2367_3, 6), m3b); //B72(8-15) B73(8-15) B76(8-15) B77(8-15) |
| 5843 | |
| 5844 | // Shuffle pattern one - right side input |
| 5845 | const __m256i rhs_mat_0145_00_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(136))); //B00(0-3) B01(0-3) B00(0-3) B01(0-3) B04(0-3) B05(0-3) B04(0-3) B05(0-3) |
| 5846 | const __m256i rhs_mat_2367_00_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_00, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(136))); //B02(0-3) B03(0-3) B02(0-3) B03(0-3) B06(0-3) B07(0-3) B06(0-3) B07(0-3) |
| 5847 | |
| 5848 | const __m256i rhs_mat_0145_01_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(136))); //B00(8-11) B01(8-11) B00(8-11) B01(8-11) B04(8-11) B05(8-11) B04(8-11) B05(8-11) |
| 5849 | const __m256i rhs_mat_2367_01_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_01, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(136))); //B02(8-11) B03(8-11) B02(8-11) B03(8-11) B06(8-11) B07(8-11) B06(8-11) B07(8-11) |
| 5850 | |
| 5851 | const __m256i rhs_mat_0145_10_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(136))); //B10(0-3) B11(0-3) B10(0-3) B11(0-3) B14(0-3) B15(0-3) B14(0-3) B15(0-3) |
| 5852 | const __m256i rhs_mat_2367_10_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_10, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(136))); //B12(0-3) B13(0-3) B12(0-3) B13(0-3) B16(0-3) B17(0-3) B16(0-3) B17(0-3) |
| 5853 | |
| 5854 | const __m256i rhs_mat_0145_11_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(136))); //B10(8-11) B11(8-11) B10(8-11) B11(8-11) B14(8-11) B15(8-11) B14(8-11) B15(8-11) |
| 5855 | const __m256i rhs_mat_2367_11_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_11, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(136))); //B12(8-11) B13(8-11) B12(8-11) B13(8-11) B16(8-11) B17(8-11) B16(8-11) B17(8-11) |
| 5856 | |
| 5857 | const __m256i rhs_mat_0145_20_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_20, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_20 ), (int)(136))); //B20(0-3) B21(0-3) B20(0-3) B21(0-3) B24(0-3) B25(0-3) B24(0-3) B25(0-3) |
| 5858 | const __m256i rhs_mat_2367_20_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_20, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_20 ), (int)(136))); //B22(0-3) B23(0-3) B22(0-3) B23(0-3) B26(0-3) B27(0-3) B26(0-3) B27(0-3) |
| 5859 | |
| 5860 | const __m256i rhs_mat_0145_21_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_21, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_21 ), (int)(136))); //B20(8-11) B21(8-11) B20(8-11) B21(8-11) B24(8-11) B25(8-11) B24(8-11) B25(8-11) |
| 5861 | const __m256i rhs_mat_2367_21_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_21, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_21 ), (int)(136))); //B22(8-11) B23(8-11) B22(8-11) B23(8-11) B26(8-11) B27(8-11) B26(8-11) B27(8-11) |
| 5862 | |
| 5863 | const __m256i rhs_mat_0145_30_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_30, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_30 ), (int)(136))); //B30(0-3) B31(0-3) B30(0-3) B31(0-3) B34(0-3) B35(0-3) B34(0-3) B35(0-3) |
| 5864 | const __m256i rhs_mat_2367_30_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_30, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_30 ), (int)(136))); //B32(0-3) B33(0-3) B32(0-3) B33(0-3) B36(0-3) B37(0-3) B36(0-3) B37(0-3) |
| 5865 | |
| 5866 | const __m256i rhs_mat_0145_31_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_31, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_31 ), (int)(136))); //B30(8-11) B31(8-11) B30(8-11) B31(8-11) B34(8-11) B35(8-11) B34(8-11) B35(8-11 |
| 5867 | const __m256i rhs_mat_2367_31_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_31, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_31 ), (int)(136))); //B32(8-11) B33(8-11) B32(8-11) B33(8-11) B36(8-11) B37(8-11) B36(8-11) B37(8-11) |
| 5868 | |
| 5869 | const __m256i rhs_mat_0145_40_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_40, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_40 ), (int)(136))); //B40(0-3) B41(0-3) B40(0-3) B41(0-3) B44(0-3) B45(0-3) B44(0-3) B45(0-3) |
| 5870 | const __m256i rhs_mat_2367_40_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_40, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_40 ), (int)(136))); //B42(0-3) B43(0-3) B42(0-3) B43(0-3) B46(0-3) B47(0-3) B46(0-3) B47(0-3) |
| 5871 | |
| 5872 | const __m256i rhs_mat_0145_41_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_41, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_41 ), (int)(136))); //B40(8-11) B41(8-11) B40(8-11) B41(8-11) B44(8-11) B45(8-11) B44(8-11) B45(8-11) |
| 5873 | const __m256i rhs_mat_2367_41_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_41, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_41 ), (int)(136))); //B42(8-11) B43(8-11) B42(8-11) B43(8-11) B46(8-11) B47(8-11) B46(8-11) B47(8-11) |
| 5874 | |
| 5875 | const __m256i rhs_mat_0145_50_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_50, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_50 ), (int)(136))); //B50(0-3) B51(0-3) B50(0-3) B51(0-3) B54(0-3) B55(0-3) B54(0-3) B55(0-3) |
| 5876 | const __m256i rhs_mat_2367_50_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_50, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_50 ), (int)(136))); //B52(0-3) B53(0-3) B52(0-3) B53(0-3) B56(0-3) B57(0-3) B56(0-3) B57(0-3) |
| 5877 | |
| 5878 | const __m256i rhs_mat_0145_51_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_51, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_51 ), (int)(136))); //B50(8-11) B51(8-11) B50(8-11) B51(8-11) B54(8-11) B55(8-11) B54(8-11) B55(8-11) |
| 5879 | const __m256i rhs_mat_2367_51_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_51, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_51 ), (int)(136))); //B52(8-11) B53(8-11) B52(8-11) B53(8-11) B56(8-11) B57(8-11) B56(8-11) B57(8-11) |
| 5880 | |
| 5881 | const __m256i rhs_mat_0145_60_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_60, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_60 ), (int)(136))); //B60(0-3) B61(0-3) B60(0-3) B61(0-3) B64(0-3) B65(0-3) B64(0-3) B65(0-3) |
| 5882 | const __m256i rhs_mat_2367_60_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_60, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_60 ), (int)(136))); //B62(0-3) B63(0-3) B62(0-3) B63(0-3) B66(0-3) B67(0-3) B66(0-3) B67(0-3) |
| 5883 | |
| 5884 | const __m256i rhs_mat_0145_61_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_61, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_61 ), (int)(136))); //B60(8-11) B61(8-11) B60(8-11) B61(8-11) B64(8-11) B65(8-11) B64(8-11) B65(8-11) |
| 5885 | const __m256i rhs_mat_2367_61_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_61, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_61 ), (int)(136))); //B62(8-11) B63(8-11) B62(8-11) B63(8-11) B66(8-11) B67(8-11) B66(8-11) B67(8-11) |
| 5886 | |
| 5887 | const __m256i rhs_mat_0145_70_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_70, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_70 ), (int)(136))); //B70(0-3) B71(0-3) B70(0-3) B71(0-3) B74(0-3) B75(0-3) B74(0-3) B75(0-3) |
| 5888 | const __m256i rhs_mat_2367_70_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_70, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_70 ), (int)(136))); //B72(0-3) B73(0-3) B72(0-3) B73(0-3) B76(0-3) B77(0-3) B76(0-3) B77(0-3) |
| 5889 | |
| 5890 | const __m256i rhs_mat_0145_71_sp1 = _mm256_shuffle_epi32(rhs_mat_0145_71, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_71 ), (int)(136))); //B70(8-11) B71(8-11) B70(8-11) B71(8-11) B74(8-11) B75(8-11) B74(8-11) B75(8-11) |
| 5891 | const __m256i rhs_mat_2367_71_sp1 = _mm256_shuffle_epi32(rhs_mat_2367_71, 136)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_71 ), (int)(136))); //B72(8-11) B73(8-11) B72(8-11) B73(8-11) B76(8-11) B77(8-11) B76(8-11) B77(8-11) |
| 5892 | |
| 5893 | |
| 5894 | // Shuffle pattern two - right side input |
| 5895 | const __m256i rhs_mat_0145_00_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_00 ), (int)(221))); //B00(4-7) B01(4-7) B00(4-7) B01(4-7) B04(4-7) B05(4-7) B04(4-7) B05(4-7) |
| 5896 | const __m256i rhs_mat_2367_00_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_00, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_00 ), (int)(221))); //B02(4-7) B03(4-7) B02(4-7) B03(4-7) B06(4-7) B07(4-7) B06(4-7) B07(4-7) |
| 5897 | |
| 5898 | const __m256i rhs_mat_0145_01_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_01 ), (int)(221))); //B00(12-15) B01(12-15) B00(12-15) B01(12-15) B04(12-15) B05(12-15) B04(12-15) B05(12-15) |
| 5899 | const __m256i rhs_mat_2367_01_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_01, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_01 ), (int)(221))); //B02(12-15) B03(12-15) B02(12-15) B03(12-15) B06(12-15) B07(12-15) B06(12-15) B07(12-15) |
| 5900 | |
| 5901 | const __m256i rhs_mat_0145_10_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_10 ), (int)(221))); //B10(4-7) B11(4-7) B10(4-7) B11(4-7) B14(4-7) B15(4-7) B14(4-7) B15(4-7) |
| 5902 | const __m256i rhs_mat_2367_10_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_10, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_10 ), (int)(221))); //B12(4-7) B13(4-7) B12(4-7) B13(4-7) B16(4-7) B17(4-7) B16(4-7) B17(4-7) |
| 5903 | |
| 5904 | const __m256i rhs_mat_0145_11_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_11 ), (int)(221))); //B10(12-15) B11(12-15) B10(12-15) B11(12-15) B14(12-15) B15(12-15) B14(12-15) B15(12-15) |
| 5905 | const __m256i rhs_mat_2367_11_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_11, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_11 ), (int)(221))); //B12(12-15) B13(12-15) B12(12-15) B13(12-15) B16(12-15) B17(12-15) B16(12-15) B17(12-15) |
| 5906 | |
| 5907 | const __m256i rhs_mat_0145_20_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_20, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_20 ), (int)(221))); //B20(4-7) B21(4-7) B20(4-7) B21(4-7) B24(4-7) B25(4-7) B24(4-7) B25(4-7) |
| 5908 | const __m256i rhs_mat_2367_20_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_20, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_20 ), (int)(221))); //B22(4-7) B23(4-7) B22(4-7) B23(4-7) B26(4-7) B27(4-7) B26(4-7) B27(4-7) |
| 5909 | |
| 5910 | const __m256i rhs_mat_0145_21_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_21, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_21 ), (int)(221))); //B20(12-15) B21(12-15) B20(12-15) B21(12-15) B24(12-15) B25(12-15) B24(12-15) B25(12-15) |
| 5911 | const __m256i rhs_mat_2367_21_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_21, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_21 ), (int)(221))); //B22(12-15) B23(12-15) B22(12-15) B23(12-15) B26(12-15) B27(12-15) B26(12-15) B27(12-15) |
| 5912 | |
| 5913 | const __m256i rhs_mat_0145_30_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_30, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_30 ), (int)(221))); //B30(4-7) B31(4-7) B30(4-7) B31(4-7) B34(4-7) B35(4-7) B34(4-7) B35(4-7) |
| 5914 | const __m256i rhs_mat_2367_30_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_30, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_30 ), (int)(221))); //B32(4-7) B33(4-7) B32(4-7) B33(4-7) B36(4-7) B37(4-7) B36(4-7) B37(4-7) |
| 5915 | |
| 5916 | const __m256i rhs_mat_0145_31_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_31, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_31 ), (int)(221))); //B30(12-15) B31(12-15) B30(12-15) B31(12-15) B34(12-15) B35(12-15) B34(12-15) B35(12-15) |
| 5917 | const __m256i rhs_mat_2367_31_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_31, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_31 ), (int)(221))); //B32(12-15) B33(12-15) B32(12-15) B33(12-15) B36(12-15) B37(12-15) B36(12-15) B37(12-15) |
| 5918 | |
| 5919 | const __m256i rhs_mat_0145_40_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_40, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_40 ), (int)(221))); //B40(4-7) B41(4-7) B40(4-7) B41(4-7) B44(4-7) B45(4-7) B44(4-7) B45(4-7) |
| 5920 | const __m256i rhs_mat_2367_40_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_40, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_40 ), (int)(221))); //B42(4-7) B43(4-7) B42(4-7) B43(4-7) B46(4-7) B47(4-7) B46(4-7) B47(4-7) |
| 5921 | |
| 5922 | const __m256i rhs_mat_0145_41_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_41, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_41 ), (int)(221))); //B40(12-15) B41(12-15) B40(12-15) B41(12-15) B44(12-15) B45(12-15) B44(12-15) B45(12-15) |
| 5923 | const __m256i rhs_mat_2367_41_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_41, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_41 ), (int)(221))); //B42(12-15) B43(12-15) B42(12-15) B43(12-15) B46(12-15) B47(12-15) B46(12-15) B47(12-15) |
| 5924 | |
| 5925 | const __m256i rhs_mat_0145_50_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_50, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_50 ), (int)(221))); //B50(4-7) B51(4-7) B50(4-7) B51(4-7) B54(4-7) B55(4-7) B54(4-7) B55(4-7) |
| 5926 | const __m256i rhs_mat_2367_50_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_50, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_50 ), (int)(221))); //B52(4-7) B53(4-7) B52(4-7) B53(4-7) B56(4-7) B57(4-7) B56(4-7) B57(4-7) |
| 5927 | |
| 5928 | const __m256i rhs_mat_0145_51_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_51, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_51 ), (int)(221))); //B50(12-15) B51(12-15) B50(12-15) B51(12-15) B54(12-15) B55(12-15) B54(12-15) B55(12-15) |
| 5929 | const __m256i rhs_mat_2367_51_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_51, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_51 ), (int)(221))); //B52(12-15) B53(12-15) B52(12-15) B53(12-15) B56(12-15) B57(12-15) B56(12-15) B57(12-15) |
| 5930 | |
| 5931 | const __m256i rhs_mat_0145_60_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_60, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_60 ), (int)(221))); //B60(4-7) B61(4-7) B60(4-7) B61(4-7) B64(4-7) B65(4-7) B64(4-7) B65(4-7) |
| 5932 | const __m256i rhs_mat_2367_60_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_60, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_60 ), (int)(221))); //B62(4-7) B63(4-7) B62(4-7) B63(4-7) B66(4-7) B67(4-7) B66(4-7) B67(4-7) |
| 5933 | |
| 5934 | const __m256i rhs_mat_0145_61_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_61, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_61 ), (int)(221))); //B60(12-15) B61(12-15) B60(12-15) B61(12-15) B64(12-15) B65(12-15) B64(12-15) B65(12-15) |
| 5935 | const __m256i rhs_mat_2367_61_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_61, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_61 ), (int)(221))); //B62(12-15) B63(12-15) B62(12-15) B63(12-15) B66(12-15) B67(12-15) B66(12-15) B67(12-15) |
| 5936 | |
| 5937 | const __m256i rhs_mat_0145_70_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_70, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_70 ), (int)(221))); //B70(4-7) B71(4-7) B70(4-7) B71(4-7) B74(4-7) B75(4-7) B74(4-7) B75(4-7) |
| 5938 | const __m256i rhs_mat_2367_70_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_70, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_70 ), (int)(221))); //B72(4-7) B73(4-7) B72(4-7) B73(4-7) B76(4-7) B77(4-7) B76(4-7) B77(4-7) |
| 5939 | |
| 5940 | const __m256i rhs_mat_0145_71_sp2 = _mm256_shuffle_epi32(rhs_mat_0145_71, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_0145_71 ), (int)(221))); //B70(12-15) B71(12-15) B70(12-15) B71(12-15) B74(12-15) B75(12-15) B74(12-15) B75(12-15) |
| 5941 | const __m256i rhs_mat_2367_71_sp2 = _mm256_shuffle_epi32(rhs_mat_2367_71, 221)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(rhs_mat_2367_71 ), (int)(221))); //B72(12-15) B73(12-15) B72(12-15) B73(12-15) B76(12-15) B77(12-15) B76(12-15) B77(12-15) |
| 5942 | |
| 5943 | |
| 5944 | //Scales and Mins of corresponding sub blocks from different Q2_K structures are stored together |
| 5945 | //s00 m00 s01 m01 s10 m10 s11 m11 s20 m20 s21 m21 s30 m30 s31 m31 s40 m40 s41 m41 s50 m50 s51 m51 s60 m60 s61 m61 s70 m70 s71 m71 |
| 5946 | |
| 5947 | // Combine mins and scales for sub-blocks: 0-1, 2-3, 4-5, 6-7 in the sb loop |
| 5948 | const __m128i mins_and_scales_01 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + sb * 64)); |
| 5949 | const __m128i mins_and_scales_23 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 16 + sb * 64)); |
| 5950 | const __m128i mins_and_scales_45 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 32 + sb * 64)); |
| 5951 | const __m128i mins_and_scales_67 = _mm_loadu_si128((const __m128i *)(b_ptr[b].scales + 48 + sb * 64)); |
| 5952 | |
| 5953 | // Extract scales which is lower half from mins_and_scales |
| 5954 | const __m128i scales_01 = _mm_and_si128(mins_and_scales_01, m4b_sse); |
| 5955 | const __m128i scales_23 = _mm_and_si128(mins_and_scales_23, m4b_sse); |
| 5956 | const __m128i scales_45 = _mm_and_si128(mins_and_scales_45, m4b_sse); |
| 5957 | const __m128i scales_67 = _mm_and_si128(mins_and_scales_67, m4b_sse); |
| 5958 | |
| 5959 | // Extract mins which is upper half from mins_and_scales |
| 5960 | const __m256i mins_01 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_01, 4), m4b_sse)); |
| 5961 | const __m256i mins_23 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_23, 4), m4b_sse)); |
| 5962 | const __m256i mins_45 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_45, 4), m4b_sse)); |
| 5963 | const __m256i mins_67 = _mm256_cvtepu8_epi16(_mm_and_si128(_mm_srli_epi16(mins_and_scales_67, 4), m4b_sse)); |
| 5964 | |
| 5965 | const __m256i scales_0 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask1_sse)); |
| 5966 | const __m256i scales_1 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_01, scalesmask2_sse)); |
| 5967 | |
| 5968 | const __m256i scales_2 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask1_sse)); |
| 5969 | const __m256i scales_3 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_23, scalesmask2_sse)); |
| 5970 | |
| 5971 | const __m256i scales_4 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask1_sse)); |
| 5972 | const __m256i scales_5 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_45, scalesmask2_sse)); |
| 5973 | |
| 5974 | const __m256i scales_6 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask1_sse)); |
| 5975 | const __m256i scales_7 = _mm256_cvtepu8_epi16(_mm_shuffle_epi8(scales_67, scalesmask2_sse)); |
| 5976 | |
| 5977 | const __m256i scale_0145_0 = _mm256_shuffle_epi32(scales_0, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(68))); |
| 5978 | const __m256i scale_2367_0 = _mm256_shuffle_epi32(scales_0, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_0 ), (int)(238))); |
| 5979 | |
| 5980 | const __m256i scale_0145_1 = _mm256_shuffle_epi32(scales_1, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(68))); |
| 5981 | const __m256i scale_2367_1 = _mm256_shuffle_epi32(scales_1, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_1 ), (int)(238))); |
| 5982 | |
| 5983 | const __m256i scale_0145_2 = _mm256_shuffle_epi32(scales_2, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_2 ), (int)(68))); |
| 5984 | const __m256i scale_2367_2 = _mm256_shuffle_epi32(scales_2, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_2 ), (int)(238))); |
| 5985 | |
| 5986 | const __m256i scale_0145_3 = _mm256_shuffle_epi32(scales_3, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_3 ), (int)(68))); |
| 5987 | const __m256i scale_2367_3 = _mm256_shuffle_epi32(scales_3, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_3 ), (int)(238))); |
| 5988 | |
| 5989 | const __m256i scale_0145_4 = _mm256_shuffle_epi32(scales_4, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_4 ), (int)(68))); |
| 5990 | const __m256i scale_2367_4 = _mm256_shuffle_epi32(scales_4, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_4 ), (int)(238))); |
| 5991 | |
| 5992 | const __m256i scale_0145_5 = _mm256_shuffle_epi32(scales_5, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_5 ), (int)(68))); |
| 5993 | const __m256i scale_2367_5 = _mm256_shuffle_epi32(scales_5, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_5 ), (int)(238))); |
| 5994 | |
| 5995 | const __m256i scale_0145_6 = _mm256_shuffle_epi32(scales_6, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_6 ), (int)(68))); |
| 5996 | const __m256i scale_2367_6 = _mm256_shuffle_epi32(scales_6, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_6 ), (int)(238))); |
| 5997 | |
| 5998 | const __m256i scale_0145_7 = _mm256_shuffle_epi32(scales_7, 68)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_7 ), (int)(68))); |
| 5999 | const __m256i scale_2367_7 = _mm256_shuffle_epi32(scales_7, 238)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(scales_7 ), (int)(238))); |
| 6000 | |
| 6001 | // Load the four block_q8_k quantized values interleaved with each other in chunks of eight bytes - A0,A1,A2,A3 |
| 6002 | // Loaded as set of 128 bit vectors and repeated into a 256 bit vector |
| 6003 | __m256i lhs_mat_0123_00 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 512 * sb))); |
| 6004 | __m256i lhs_mat_01_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(0))); |
| 6005 | __m256i lhs_mat_23_00 = _mm256_permute2f128_si256(lhs_mat_0123_00, lhs_mat_0123_00, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_00 ), (__v8si)(__m256i)(lhs_mat_0123_00), (int)(17))); |
| 6006 | __m256i lhs_mat_0123_01 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 32 + 512 * sb))); |
| 6007 | __m256i lhs_mat_01_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(0))); |
| 6008 | __m256i lhs_mat_23_01 = _mm256_permute2f128_si256(lhs_mat_0123_01, lhs_mat_0123_01, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_01 ), (__v8si)(__m256i)(lhs_mat_0123_01), (int)(17))); |
| 6009 | __m256i lhs_mat_0123_10 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 64 + 512 * sb))); |
| 6010 | __m256i lhs_mat_01_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(0))); |
| 6011 | __m256i lhs_mat_23_10 = _mm256_permute2f128_si256(lhs_mat_0123_10, lhs_mat_0123_10, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_10 ), (__v8si)(__m256i)(lhs_mat_0123_10), (int)(17))); |
| 6012 | __m256i lhs_mat_0123_11 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 96 + 512 * sb))); |
| 6013 | __m256i lhs_mat_01_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(0))); |
| 6014 | __m256i lhs_mat_23_11 = _mm256_permute2f128_si256(lhs_mat_0123_11, lhs_mat_0123_11, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_11 ), (__v8si)(__m256i)(lhs_mat_0123_11), (int)(17))); |
| 6015 | __m256i lhs_mat_0123_20 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 128 + 512 * sb))); |
| 6016 | __m256i lhs_mat_01_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_20 ), (__v8si)(__m256i)(lhs_mat_0123_20), (int)(0))); |
| 6017 | __m256i lhs_mat_23_20 = _mm256_permute2f128_si256(lhs_mat_0123_20, lhs_mat_0123_20, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_20 ), (__v8si)(__m256i)(lhs_mat_0123_20), (int)(17))); |
| 6018 | __m256i lhs_mat_0123_21 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 160 + 512 * sb))); |
| 6019 | __m256i lhs_mat_01_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_21 ), (__v8si)(__m256i)(lhs_mat_0123_21), (int)(0))); |
| 6020 | __m256i lhs_mat_23_21 = _mm256_permute2f128_si256(lhs_mat_0123_21, lhs_mat_0123_21, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_21 ), (__v8si)(__m256i)(lhs_mat_0123_21), (int)(17))); |
| 6021 | __m256i lhs_mat_0123_30 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 192 + 512 * sb))); |
| 6022 | __m256i lhs_mat_01_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_30 ), (__v8si)(__m256i)(lhs_mat_0123_30), (int)(0))); |
| 6023 | __m256i lhs_mat_23_30 = _mm256_permute2f128_si256(lhs_mat_0123_30, lhs_mat_0123_30, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_30 ), (__v8si)(__m256i)(lhs_mat_0123_30), (int)(17))); |
| 6024 | __m256i lhs_mat_0123_31 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 224 + 512 * sb))); |
| 6025 | __m256i lhs_mat_01_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_31 ), (__v8si)(__m256i)(lhs_mat_0123_31), (int)(0))); |
| 6026 | __m256i lhs_mat_23_31 = _mm256_permute2f128_si256(lhs_mat_0123_31, lhs_mat_0123_31, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_31 ), (__v8si)(__m256i)(lhs_mat_0123_31), (int)(17))); |
| 6027 | |
| 6028 | __m256i lhs_mat_0123_40 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 256 + 512 * sb))); |
| 6029 | __m256i lhs_mat_01_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_40 ), (__v8si)(__m256i)(lhs_mat_0123_40), (int)(0))); |
| 6030 | __m256i lhs_mat_23_40 = _mm256_permute2f128_si256(lhs_mat_0123_40, lhs_mat_0123_40, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_40 ), (__v8si)(__m256i)(lhs_mat_0123_40), (int)(17))); |
| 6031 | __m256i lhs_mat_0123_41 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 288 + 512 * sb))); |
| 6032 | __m256i lhs_mat_01_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_41 ), (__v8si)(__m256i)(lhs_mat_0123_41), (int)(0))); |
| 6033 | __m256i lhs_mat_23_41 = _mm256_permute2f128_si256(lhs_mat_0123_41, lhs_mat_0123_41, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_41 ), (__v8si)(__m256i)(lhs_mat_0123_41), (int)(17))); |
| 6034 | __m256i lhs_mat_0123_50 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 320 + 512 * sb))); |
| 6035 | __m256i lhs_mat_01_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_50 ), (__v8si)(__m256i)(lhs_mat_0123_50), (int)(0))); |
| 6036 | __m256i lhs_mat_23_50 = _mm256_permute2f128_si256(lhs_mat_0123_50, lhs_mat_0123_50, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_50 ), (__v8si)(__m256i)(lhs_mat_0123_50), (int)(17))); |
| 6037 | __m256i lhs_mat_0123_51 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 352 + 512 * sb))); |
| 6038 | __m256i lhs_mat_01_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_51 ), (__v8si)(__m256i)(lhs_mat_0123_51), (int)(0))); |
| 6039 | __m256i lhs_mat_23_51 = _mm256_permute2f128_si256(lhs_mat_0123_51, lhs_mat_0123_51, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_51 ), (__v8si)(__m256i)(lhs_mat_0123_51), (int)(17))); |
| 6040 | __m256i lhs_mat_0123_60 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 384 + 512 * sb))); |
| 6041 | __m256i lhs_mat_01_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_60 ), (__v8si)(__m256i)(lhs_mat_0123_60), (int)(0))); |
| 6042 | __m256i lhs_mat_23_60 = _mm256_permute2f128_si256(lhs_mat_0123_60, lhs_mat_0123_60, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_60 ), (__v8si)(__m256i)(lhs_mat_0123_60), (int)(17))); |
| 6043 | __m256i lhs_mat_0123_61 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 416 + 512 * sb))); |
| 6044 | __m256i lhs_mat_01_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_61 ), (__v8si)(__m256i)(lhs_mat_0123_61), (int)(0))); |
| 6045 | __m256i lhs_mat_23_61 = _mm256_permute2f128_si256(lhs_mat_0123_61, lhs_mat_0123_61, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_61 ), (__v8si)(__m256i)(lhs_mat_0123_61), (int)(17))); |
| 6046 | __m256i lhs_mat_0123_70 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 448 + 512 * sb))); |
| 6047 | __m256i lhs_mat_01_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_70 ), (__v8si)(__m256i)(lhs_mat_0123_70), (int)(0))); |
| 6048 | __m256i lhs_mat_23_70 = _mm256_permute2f128_si256(lhs_mat_0123_70, lhs_mat_0123_70, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_70 ), (__v8si)(__m256i)(lhs_mat_0123_70), (int)(17))); |
| 6049 | __m256i lhs_mat_0123_71 = _mm256_loadu_si256((const __m256i * )((a_ptr[b].qs + 480 + 512 * sb))); |
| 6050 | __m256i lhs_mat_01_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 0)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_71 ), (__v8si)(__m256i)(lhs_mat_0123_71), (int)(0))); |
| 6051 | __m256i lhs_mat_23_71 = _mm256_permute2f128_si256(lhs_mat_0123_71, lhs_mat_0123_71, 17)((__m256i)__builtin_ia32_vperm2f128_si256((__v8si)(__m256i)(lhs_mat_0123_71 ), (__v8si)(__m256i)(lhs_mat_0123_71), (int)(17))); |
| 6052 | |
| 6053 | // Bsums are loaded for the different Q8_K blocks |
| 6054 | __m128i lhs_raw_bsums_01_0123 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 32 * sb))); |
| 6055 | __m128i lhs_raw_bsums_23_0123 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 8 + 32 * sb)); |
| 6056 | __m128i lhs_raw_bsums_01_4567 = _mm_loadu_si128((const __m128i *)((a_ptr[b].bsums + 16 + 32 * sb))); |
| 6057 | __m128i lhs_raw_bsums_23_4567 = _mm_loadu_si128((const __m128i *)(a_ptr[b].bsums + 24 + 32 * sb)); |
| 6058 | |
| 6059 | // Shuffle pattern one - left side input |
| 6060 | const __m256i lhs_mat_01_00_sp1 = _mm256_shuffle_epi32(lhs_mat_01_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(160))); //A00(0-3) A00(0-3) A01(0-3) A01(0-3) A00(0-3) A00(0-3) A01(0-3) A01(0-3) |
| 6061 | const __m256i lhs_mat_23_00_sp1 = _mm256_shuffle_epi32(lhs_mat_23_00, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(160))); //A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) A02(0-3) A03(0-3) |
| 6062 | |
| 6063 | const __m256i lhs_mat_01_01_sp1 = _mm256_shuffle_epi32(lhs_mat_01_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(160))); //A00(8-11) A00(8-11) A01(8-11) A01(8-11) A00(8-11) A00(8-11) A01(8-11) A01(8-11) |
| 6064 | const __m256i lhs_mat_23_01_sp1 = _mm256_shuffle_epi32(lhs_mat_23_01, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(160))); //A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) A02(8-11) A03(8-11) |
| 6065 | |
| 6066 | const __m256i lhs_mat_01_10_sp1 = _mm256_shuffle_epi32(lhs_mat_01_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(160))); //A10(0-3) A10(0-3) A11(0-3) A11(0-3) A10(0-3) A10(0-3) A11(0-3) A11(0-3) |
| 6067 | const __m256i lhs_mat_23_10_sp1 = _mm256_shuffle_epi32(lhs_mat_23_10, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(160))); //A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) A12(0-3) A13(0-3) |
| 6068 | |
| 6069 | const __m256i lhs_mat_01_11_sp1 = _mm256_shuffle_epi32(lhs_mat_01_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(160))); //A10(8-11) A10(8-11) A11(8-11) A11(8-11) A10(8-11) A10(8-11) A11(8-11) A11(8-11) |
| 6070 | const __m256i lhs_mat_23_11_sp1 = _mm256_shuffle_epi32(lhs_mat_23_11, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(160))); //A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) A12(8-11) A13(8-11) |
| 6071 | |
| 6072 | const __m256i lhs_mat_01_20_sp1 = _mm256_shuffle_epi32(lhs_mat_01_20, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_20 ), (int)(160))); //A20(0-3) A20(0-3) A21(0-3) A21(0-3) A20(0-3) A20(0-3) A21(0-3) A21(0-3) |
| 6073 | const __m256i lhs_mat_23_20_sp1 = _mm256_shuffle_epi32(lhs_mat_23_20, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_20 ), (int)(160))); //A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) A22(0-3) A23(0-3) |
| 6074 | |
| 6075 | const __m256i lhs_mat_01_21_sp1 = _mm256_shuffle_epi32(lhs_mat_01_21, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_21 ), (int)(160))); //A20(8-11) A20(8-11) A21(8-11) A21(8-11) A20(8-11) A20(8-11) A21(8-11) A21(8-11) |
| 6076 | const __m256i lhs_mat_23_21_sp1 = _mm256_shuffle_epi32(lhs_mat_23_21, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_21 ), (int)(160))); //A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) A22(8-11) A23(8-11) |
| 6077 | |
| 6078 | const __m256i lhs_mat_01_30_sp1 = _mm256_shuffle_epi32(lhs_mat_01_30, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_30 ), (int)(160))); //A30(0-3) A30(0-3) A31(0-3) A31(0-3) A30(0-3) A30(0-3) A31(0-3) A31(0-3) |
| 6079 | const __m256i lhs_mat_23_30_sp1 = _mm256_shuffle_epi32(lhs_mat_23_30, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_30 ), (int)(160))); //A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) A32(0-3) A33(0-3) |
| 6080 | |
| 6081 | const __m256i lhs_mat_01_31_sp1 = _mm256_shuffle_epi32(lhs_mat_01_31, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_31 ), (int)(160))); //A30(8-11) A30(8-11) A31(8-11) A31(8-11) A30(8-11) A30(8-11) A31(8-11) A31(8-11) |
| 6082 | const __m256i lhs_mat_23_31_sp1 = _mm256_shuffle_epi32(lhs_mat_23_31, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_31 ), (int)(160))); //A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) A32(8-11) A33(8-11) |
| 6083 | |
| 6084 | const __m256i lhs_mat_01_40_sp1 = _mm256_shuffle_epi32(lhs_mat_01_40, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_40 ), (int)(160))); //A40(0-3) A40(0-3) A41(0-3) A41(0-3) A40(0-3) A40(0-3) A41(0-3) A41(0-3) |
| 6085 | const __m256i lhs_mat_23_40_sp1 = _mm256_shuffle_epi32(lhs_mat_23_40, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_40 ), (int)(160))); //A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) A42(0-3) A43(0-3) |
| 6086 | |
| 6087 | const __m256i lhs_mat_01_41_sp1 = _mm256_shuffle_epi32(lhs_mat_01_41, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_41 ), (int)(160))); //A40(8-11) A40(8-11) A41(8-11) A41(8-11) A40(8-11) A40(8-11) A41(8-11) A41(8-11) |
| 6088 | const __m256i lhs_mat_23_41_sp1 = _mm256_shuffle_epi32(lhs_mat_23_41, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_41 ), (int)(160))); //A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) A42(8-11) A43(8-11) |
| 6089 | |
| 6090 | const __m256i lhs_mat_01_50_sp1 = _mm256_shuffle_epi32(lhs_mat_01_50, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_50 ), (int)(160))); //A50(0-3) A50(0-3) A51(0-3) A51(0-3) A50(0-3) A50(0-3) A51(0-3) A51(0-3) |
| 6091 | const __m256i lhs_mat_23_50_sp1 = _mm256_shuffle_epi32(lhs_mat_23_50, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_50 ), (int)(160))); //A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) A52(0-3) A53(0-3) |
| 6092 | |
| 6093 | const __m256i lhs_mat_01_51_sp1 = _mm256_shuffle_epi32(lhs_mat_01_51, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_51 ), (int)(160))); //A50(8-11) A50(8-11) A51(8-11) A51(8-11) A50(8-11) A50(8-11) A51(8-11) A51(8-11) |
| 6094 | const __m256i lhs_mat_23_51_sp1 = _mm256_shuffle_epi32(lhs_mat_23_51, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_51 ), (int)(160))); //A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) A52(8-11) A53(8-11) |
| 6095 | |
| 6096 | const __m256i lhs_mat_01_60_sp1 = _mm256_shuffle_epi32(lhs_mat_01_60, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_60 ), (int)(160))); //A60(0-3) A60(0-3) A61(0-3) A61(0-3) A60(0-3) A60(0-3) A61(0-3) A61(0-3) |
| 6097 | const __m256i lhs_mat_23_60_sp1 = _mm256_shuffle_epi32(lhs_mat_23_60, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_60 ), (int)(160))); //A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) A62(0-3) A63(0-3) |
| 6098 | |
| 6099 | const __m256i lhs_mat_01_61_sp1 = _mm256_shuffle_epi32(lhs_mat_01_61, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_61 ), (int)(160))); //A60(8-11) A60(8-11) A61(8-11) A61(8-11) A60(8-11) A60(8-11) A61(8-11) A61(8-11) |
| 6100 | const __m256i lhs_mat_23_61_sp1 = _mm256_shuffle_epi32(lhs_mat_23_61, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_61 ), (int)(160))); //A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) A62(8-11) A63(8-11) |
| 6101 | |
| 6102 | const __m256i lhs_mat_01_70_sp1 = _mm256_shuffle_epi32(lhs_mat_01_70, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_70 ), (int)(160))); //A70(0-3) A70(0-3) A71(0-3) A71(0-3) A70(0-3) A70(0-3) A71(0-3) A71(0-3) |
| 6103 | const __m256i lhs_mat_23_70_sp1 = _mm256_shuffle_epi32(lhs_mat_23_70, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_70 ), (int)(160))); //A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) A72(0-3) A73(0-3) |
| 6104 | |
| 6105 | const __m256i lhs_mat_01_71_sp1 = _mm256_shuffle_epi32(lhs_mat_01_71, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_71 ), (int)(160))); //A70(8-11) A70(8-11) A71(8-11) A71(8-11) A70(8-11) A70(8-11) A71(8-11) A71(8-11) |
| 6106 | const __m256i lhs_mat_23_71_sp1 = _mm256_shuffle_epi32(lhs_mat_23_71, 160)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_71 ), (int)(160))); //A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) A72(8-11) A73(8-11) |
| 6107 | |
| 6108 | // Shuffle pattern two- left side input |
| 6109 | const __m256i lhs_mat_01_00_sp2 = _mm256_shuffle_epi32(lhs_mat_01_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_00 ), (int)(245))); //A00(4-7) A00(4-7) A01(4-7) A01(4-7) A00(4-7) A00(4-7) A01(4-7) A01(4-7) |
| 6110 | const __m256i lhs_mat_23_00_sp2 = _mm256_shuffle_epi32(lhs_mat_23_00, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_00 ), (int)(245))); //A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) A02(4-7) A03(4-7) |
| 6111 | |
| 6112 | const __m256i lhs_mat_01_01_sp2 = _mm256_shuffle_epi32(lhs_mat_01_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_01 ), (int)(245))); //A00(12-15) A00(12-15) A01(12-15) A01(12-15) A00(12-15) A00(12-15) A01(12-15) A01(12-15) |
| 6113 | const __m256i lhs_mat_23_01_sp2 = _mm256_shuffle_epi32(lhs_mat_23_01, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_01 ), (int)(245))); //A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) A02(12-15) A03(12-15) |
| 6114 | |
| 6115 | const __m256i lhs_mat_01_10_sp2 = _mm256_shuffle_epi32(lhs_mat_01_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_10 ), (int)(245))); //A10(4-7) A10(4-7) A11(4-7) A11(4-7) A10(4-7) A10(4-7) A11(4-7) A11(4-7) |
| 6116 | const __m256i lhs_mat_23_10_sp2 = _mm256_shuffle_epi32(lhs_mat_23_10, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_10 ), (int)(245))); //A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) A12(4-7) A13(4-7) |
| 6117 | |
| 6118 | const __m256i lhs_mat_01_11_sp2 = _mm256_shuffle_epi32(lhs_mat_01_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_11 ), (int)(245))); //A10(12-15) A10(12-15) A11(12-15) A11(12-15) A10(12-15) A10(12-15) A11(12-15) A11(12-15) |
| 6119 | const __m256i lhs_mat_23_11_sp2 = _mm256_shuffle_epi32(lhs_mat_23_11, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_11 ), (int)(245))); //A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) A12(12-15) A13(12-15) |
| 6120 | |
| 6121 | const __m256i lhs_mat_01_20_sp2 = _mm256_shuffle_epi32(lhs_mat_01_20, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_20 ), (int)(245))); //A20(4-7) A20(4-7) A21(4-7) A21(4-7) A20(4-7) A20(4-7) A21(4-7) A21(4-7) |
| 6122 | const __m256i lhs_mat_23_20_sp2 = _mm256_shuffle_epi32(lhs_mat_23_20, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_20 ), (int)(245))); //A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) A22(4-7) A23(4-7) |
| 6123 | |
| 6124 | const __m256i lhs_mat_01_21_sp2 = _mm256_shuffle_epi32(lhs_mat_01_21, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_21 ), (int)(245))); //A20(12-15) A20(12-15) A21(12-15) A21(12-15) A20(12-15) A20(12-15) A21(12-15) A21(12-15) |
| 6125 | const __m256i lhs_mat_23_21_sp2 = _mm256_shuffle_epi32(lhs_mat_23_21, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_21 ), (int)(245))); //A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) A22(12-15) A23(12-15) |
| 6126 | |
| 6127 | const __m256i lhs_mat_01_30_sp2 = _mm256_shuffle_epi32(lhs_mat_01_30, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_30 ), (int)(245))); //A30(4-7) A30(4-7) A31(4-7) A31(4-7) A30(4-7) A30(4-7) A31(4-7) A31(4-7) |
| 6128 | const __m256i lhs_mat_23_30_sp2 = _mm256_shuffle_epi32(lhs_mat_23_30, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_30 ), (int)(245))); //A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) A32(4-7) A33(4-7) |
| 6129 | |
| 6130 | const __m256i lhs_mat_01_31_sp2 = _mm256_shuffle_epi32(lhs_mat_01_31, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_31 ), (int)(245))); //A30(12-15) A30(12-15) A31(12-15) A31(12-15) A30(12-15) A30(12-15) A31(12-15) A31(12-15) |
| 6131 | const __m256i lhs_mat_23_31_sp2 = _mm256_shuffle_epi32(lhs_mat_23_31, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_31 ), (int)(245))); //A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) A32(12-15) A33(12-15) |
| 6132 | |
| 6133 | const __m256i lhs_mat_01_40_sp2 = _mm256_shuffle_epi32(lhs_mat_01_40, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_40 ), (int)(245))); //A40(4-7) A40(4-7) A41(4-7) A41(4-7) A40(4-7) A40(4-7) A41(4-7) A41(4-7) |
| 6134 | const __m256i lhs_mat_23_40_sp2 = _mm256_shuffle_epi32(lhs_mat_23_40, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_40 ), (int)(245))); //A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) A42(4-7) A43(4-7) |
| 6135 | |
| 6136 | const __m256i lhs_mat_01_41_sp2 = _mm256_shuffle_epi32(lhs_mat_01_41, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_41 ), (int)(245))); //A40(12-15) A40(12-15) A41(12-15) A41(12-15) A40(12-15) A40(12-15) A41(12-15) A41(12-15) |
| 6137 | const __m256i lhs_mat_23_41_sp2 = _mm256_shuffle_epi32(lhs_mat_23_41, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_41 ), (int)(245))); //A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) A42(12-15) A43(12-15) |
| 6138 | |
| 6139 | const __m256i lhs_mat_01_50_sp2 = _mm256_shuffle_epi32(lhs_mat_01_50, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_50 ), (int)(245))); //A50(4-7) A50(4-7) A51(4-7) A51(4-7) A50(4-7) A50(4-7) A51(4-7) A51(4-7) |
| 6140 | const __m256i lhs_mat_23_50_sp2 = _mm256_shuffle_epi32(lhs_mat_23_50, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_50 ), (int)(245))); //A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) A52(4-7) A53(4-7) |
| 6141 | |
| 6142 | const __m256i lhs_mat_01_51_sp2 = _mm256_shuffle_epi32(lhs_mat_01_51, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_51 ), (int)(245))); //A50(12-15) A50(12-15) A51(12-15) A51(12-15) A50(12-15) A50(12-15) A51(12-15) A51(12-15) |
| 6143 | const __m256i lhs_mat_23_51_sp2 = _mm256_shuffle_epi32(lhs_mat_23_51, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_51 ), (int)(245))); //A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) A52(12-15) A53(12-15) |
| 6144 | |
| 6145 | const __m256i lhs_mat_01_60_sp2 = _mm256_shuffle_epi32(lhs_mat_01_60, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_60 ), (int)(245))); //A60(4-7) A60(4-7) A61(4-7) A61(4-7) A60(4-7) A60(4-7) A61(4-7) A61(4-7) |
| 6146 | const __m256i lhs_mat_23_60_sp2 = _mm256_shuffle_epi32(lhs_mat_23_60, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_60 ), (int)(245))); //A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) A62(4-7) A63(4-7) |
| 6147 | |
| 6148 | const __m256i lhs_mat_01_61_sp2 = _mm256_shuffle_epi32(lhs_mat_01_61, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_61 ), (int)(245))); //A60(12-15) A60(12-15) A61(12-15) A61(12-15) A60(12-15) A60(12-15) A61(12-15) A61(12-15) |
| 6149 | const __m256i lhs_mat_23_61_sp2 = _mm256_shuffle_epi32(lhs_mat_23_61, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_61 ), (int)(245))); //A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) A62(12-15) A63(12-15) |
| 6150 | |
| 6151 | const __m256i lhs_mat_01_70_sp2 = _mm256_shuffle_epi32(lhs_mat_01_70, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_70 ), (int)(245))); //A70(4-7) A70(4-7) A71(4-7) A71(4-7) A70(4-7) A70(4-7) A71(4-7) A71(4-7) |
| 6152 | const __m256i lhs_mat_23_70_sp2 = _mm256_shuffle_epi32(lhs_mat_23_70, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_70 ), (int)(245))); //A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) A72(4-7) A73(4-7) |
| 6153 | |
| 6154 | const __m256i lhs_mat_01_71_sp2 = _mm256_shuffle_epi32(lhs_mat_01_71, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_01_71 ), (int)(245))); //A70(12-15) A70(12-15) A71(12-15) A71(12-15) A70(12-15) A70(12-15) A71(12-15) A71(12-15) |
| 6155 | const __m256i lhs_mat_23_71_sp2 = _mm256_shuffle_epi32(lhs_mat_23_71, 245)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_mat_23_71 ), (int)(245))); //A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) A72(12-15) A73(12-15) |
| 6156 | |
| 6157 | // The values arranged in shuffle patterns are operated with dot product operation within 32 bit lane i.e corresponding bytes and multiplied and added into 32 bit integers within 32 bit lane |
| 6158 | __m256i iacc_mat_00_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_01_01_sp1)); |
| 6159 | __m256i iacc_mat_01_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_01_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_01_01_sp1)); |
| 6160 | |
| 6161 | __m256i iacc_mat_10_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_0145_01_sp1, lhs_mat_23_01_sp1)); |
| 6162 | __m256i iacc_mat_11_0_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp1, lhs_mat_23_00_sp1),_mm256_maddubs_epi16(rhs_mat_2367_01_sp1, lhs_mat_23_01_sp1)); |
| 6163 | |
| 6164 | __m256i iacc_mat_00_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_01_11_sp1)); |
| 6165 | __m256i iacc_mat_01_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_01_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_01_11_sp1)); |
| 6166 | |
| 6167 | __m256i iacc_mat_10_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_0145_11_sp1, lhs_mat_23_11_sp1)); |
| 6168 | __m256i iacc_mat_11_1_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp1, lhs_mat_23_10_sp1),_mm256_maddubs_epi16(rhs_mat_2367_11_sp1, lhs_mat_23_11_sp1)); |
| 6169 | |
| 6170 | __m256i iacc_mat_00_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_01_21_sp1)); |
| 6171 | __m256i iacc_mat_01_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_01_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_01_21_sp1)); |
| 6172 | |
| 6173 | __m256i iacc_mat_10_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_0145_21_sp1, lhs_mat_23_21_sp1)); |
| 6174 | __m256i iacc_mat_11_2_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp1, lhs_mat_23_20_sp1),_mm256_maddubs_epi16(rhs_mat_2367_21_sp1, lhs_mat_23_21_sp1)); |
| 6175 | |
| 6176 | __m256i iacc_mat_00_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_01_31_sp1)); |
| 6177 | __m256i iacc_mat_01_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_01_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_01_31_sp1)); |
| 6178 | |
| 6179 | __m256i iacc_mat_10_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_0145_31_sp1, lhs_mat_23_31_sp1)); |
| 6180 | __m256i iacc_mat_11_3_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp1, lhs_mat_23_30_sp1),_mm256_maddubs_epi16(rhs_mat_2367_31_sp1, lhs_mat_23_31_sp1)); |
| 6181 | |
| 6182 | __m256i iacc_mat_00_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_01_41_sp1)); |
| 6183 | __m256i iacc_mat_01_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_01_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_01_41_sp1)); |
| 6184 | |
| 6185 | __m256i iacc_mat_10_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_0145_41_sp1, lhs_mat_23_41_sp1)); |
| 6186 | __m256i iacc_mat_11_4_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp1, lhs_mat_23_40_sp1),_mm256_maddubs_epi16(rhs_mat_2367_41_sp1, lhs_mat_23_41_sp1)); |
| 6187 | |
| 6188 | __m256i iacc_mat_00_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_01_51_sp1)); |
| 6189 | __m256i iacc_mat_01_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_01_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_01_51_sp1)); |
| 6190 | |
| 6191 | __m256i iacc_mat_10_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_0145_51_sp1, lhs_mat_23_51_sp1)); |
| 6192 | __m256i iacc_mat_11_5_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp1, lhs_mat_23_50_sp1),_mm256_maddubs_epi16(rhs_mat_2367_51_sp1, lhs_mat_23_51_sp1)); |
| 6193 | |
| 6194 | __m256i iacc_mat_00_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_01_61_sp1)); |
| 6195 | __m256i iacc_mat_01_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_01_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_01_61_sp1)); |
| 6196 | |
| 6197 | __m256i iacc_mat_10_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_0145_61_sp1, lhs_mat_23_61_sp1)); |
| 6198 | __m256i iacc_mat_11_6_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp1, lhs_mat_23_60_sp1),_mm256_maddubs_epi16(rhs_mat_2367_61_sp1, lhs_mat_23_61_sp1)); |
| 6199 | |
| 6200 | __m256i iacc_mat_00_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_01_71_sp1)); |
| 6201 | __m256i iacc_mat_01_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_01_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_01_71_sp1)); |
| 6202 | |
| 6203 | __m256i iacc_mat_10_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_0145_71_sp1, lhs_mat_23_71_sp1)); |
| 6204 | __m256i iacc_mat_11_7_sp1 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp1, lhs_mat_23_70_sp1),_mm256_maddubs_epi16(rhs_mat_2367_71_sp1, lhs_mat_23_71_sp1)); |
| 6205 | |
| 6206 | |
| 6207 | __m256i iacc_mat_00_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_01_01_sp2)); |
| 6208 | __m256i iacc_mat_01_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_01_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_01_01_sp2)); |
| 6209 | |
| 6210 | __m256i iacc_mat_10_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_0145_01_sp2, lhs_mat_23_01_sp2)); |
| 6211 | __m256i iacc_mat_11_0_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_00_sp2, lhs_mat_23_00_sp2),_mm256_maddubs_epi16(rhs_mat_2367_01_sp2, lhs_mat_23_01_sp2)); |
| 6212 | |
| 6213 | __m256i iacc_mat_00_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_01_11_sp2)); |
| 6214 | __m256i iacc_mat_01_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_01_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_01_11_sp2)); |
| 6215 | |
| 6216 | __m256i iacc_mat_10_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_0145_11_sp2, lhs_mat_23_11_sp2)); |
| 6217 | __m256i iacc_mat_11_1_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_10_sp2, lhs_mat_23_10_sp2),_mm256_maddubs_epi16(rhs_mat_2367_11_sp2, lhs_mat_23_11_sp2)); |
| 6218 | |
| 6219 | __m256i iacc_mat_00_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_01_21_sp2)); |
| 6220 | __m256i iacc_mat_01_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_01_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_01_21_sp2)); |
| 6221 | |
| 6222 | __m256i iacc_mat_10_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_0145_21_sp2, lhs_mat_23_21_sp2)); |
| 6223 | __m256i iacc_mat_11_2_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_20_sp2, lhs_mat_23_20_sp2),_mm256_maddubs_epi16(rhs_mat_2367_21_sp2, lhs_mat_23_21_sp2)); |
| 6224 | |
| 6225 | __m256i iacc_mat_00_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_01_31_sp2)); |
| 6226 | __m256i iacc_mat_01_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_01_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_01_31_sp2)); |
| 6227 | |
| 6228 | __m256i iacc_mat_10_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_0145_31_sp2, lhs_mat_23_31_sp2)); |
| 6229 | __m256i iacc_mat_11_3_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_30_sp2, lhs_mat_23_30_sp2),_mm256_maddubs_epi16(rhs_mat_2367_31_sp2, lhs_mat_23_31_sp2)); |
| 6230 | |
| 6231 | __m256i iacc_mat_00_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_01_41_sp2)); |
| 6232 | __m256i iacc_mat_01_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_01_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_01_41_sp2)); |
| 6233 | |
| 6234 | __m256i iacc_mat_10_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_0145_41_sp2, lhs_mat_23_41_sp2)); |
| 6235 | __m256i iacc_mat_11_4_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_40_sp2, lhs_mat_23_40_sp2),_mm256_maddubs_epi16(rhs_mat_2367_41_sp2, lhs_mat_23_41_sp2)); |
| 6236 | |
| 6237 | __m256i iacc_mat_00_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_01_51_sp2)); |
| 6238 | __m256i iacc_mat_01_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_01_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_01_51_sp2)); |
| 6239 | |
| 6240 | __m256i iacc_mat_10_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_0145_51_sp2, lhs_mat_23_51_sp2)); |
| 6241 | __m256i iacc_mat_11_5_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_50_sp2, lhs_mat_23_50_sp2),_mm256_maddubs_epi16(rhs_mat_2367_51_sp2, lhs_mat_23_51_sp2)); |
| 6242 | |
| 6243 | __m256i iacc_mat_00_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_01_61_sp2)); |
| 6244 | __m256i iacc_mat_01_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_01_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_01_61_sp2)); |
| 6245 | |
| 6246 | __m256i iacc_mat_10_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_0145_61_sp2, lhs_mat_23_61_sp2)); |
| 6247 | __m256i iacc_mat_11_6_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_60_sp2, lhs_mat_23_60_sp2),_mm256_maddubs_epi16(rhs_mat_2367_61_sp2, lhs_mat_23_61_sp2)); |
| 6248 | |
| 6249 | __m256i iacc_mat_00_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_01_71_sp2)); |
| 6250 | __m256i iacc_mat_01_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_01_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_01_71_sp2)); |
| 6251 | |
| 6252 | __m256i iacc_mat_10_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_0145_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_0145_71_sp2, lhs_mat_23_71_sp2)); |
| 6253 | __m256i iacc_mat_11_7_sp2 = _mm256_add_epi16(_mm256_maddubs_epi16(rhs_mat_2367_70_sp2, lhs_mat_23_70_sp2),_mm256_maddubs_epi16(rhs_mat_2367_71_sp2, lhs_mat_23_71_sp2)); |
| 6254 | |
| 6255 | // Combine results from both shuffle patterns for each output block. |
| 6256 | __m256i iacc_mat_00_0 = _mm256_add_epi16(iacc_mat_00_0_sp1, iacc_mat_00_0_sp2); |
| 6257 | __m256i iacc_mat_01_0 = _mm256_add_epi16(iacc_mat_01_0_sp1, iacc_mat_01_0_sp2); |
| 6258 | __m256i iacc_mat_10_0 = _mm256_add_epi16(iacc_mat_10_0_sp1, iacc_mat_10_0_sp2); |
| 6259 | __m256i iacc_mat_11_0 = _mm256_add_epi16(iacc_mat_11_0_sp1, iacc_mat_11_0_sp2); |
| 6260 | |
| 6261 | __m256i iacc_mat_00_1 = _mm256_add_epi16(iacc_mat_00_1_sp1, iacc_mat_00_1_sp2); |
| 6262 | __m256i iacc_mat_01_1 = _mm256_add_epi16(iacc_mat_01_1_sp1, iacc_mat_01_1_sp2); |
| 6263 | __m256i iacc_mat_10_1 = _mm256_add_epi16(iacc_mat_10_1_sp1, iacc_mat_10_1_sp2); |
| 6264 | __m256i iacc_mat_11_1 = _mm256_add_epi16(iacc_mat_11_1_sp1, iacc_mat_11_1_sp2); |
| 6265 | |
| 6266 | __m256i iacc_mat_00_2 = _mm256_add_epi16(iacc_mat_00_2_sp1, iacc_mat_00_2_sp2); |
| 6267 | __m256i iacc_mat_01_2 = _mm256_add_epi16(iacc_mat_01_2_sp1, iacc_mat_01_2_sp2); |
| 6268 | __m256i iacc_mat_10_2 = _mm256_add_epi16(iacc_mat_10_2_sp1, iacc_mat_10_2_sp2); |
| 6269 | __m256i iacc_mat_11_2 = _mm256_add_epi16(iacc_mat_11_2_sp1, iacc_mat_11_2_sp2); |
| 6270 | |
| 6271 | __m256i iacc_mat_00_3 = _mm256_add_epi16(iacc_mat_00_3_sp1, iacc_mat_00_3_sp2); |
| 6272 | __m256i iacc_mat_01_3 = _mm256_add_epi16(iacc_mat_01_3_sp1, iacc_mat_01_3_sp2); |
| 6273 | __m256i iacc_mat_10_3 = _mm256_add_epi16(iacc_mat_10_3_sp1, iacc_mat_10_3_sp2); |
| 6274 | __m256i iacc_mat_11_3 = _mm256_add_epi16(iacc_mat_11_3_sp1, iacc_mat_11_3_sp2); |
| 6275 | |
| 6276 | __m256i iacc_mat_00_4 = _mm256_add_epi16(iacc_mat_00_4_sp1, iacc_mat_00_4_sp2); |
| 6277 | __m256i iacc_mat_01_4 = _mm256_add_epi16(iacc_mat_01_4_sp1, iacc_mat_01_4_sp2); |
| 6278 | __m256i iacc_mat_10_4 = _mm256_add_epi16(iacc_mat_10_4_sp1, iacc_mat_10_4_sp2); |
| 6279 | __m256i iacc_mat_11_4 = _mm256_add_epi16(iacc_mat_11_4_sp1, iacc_mat_11_4_sp2); |
| 6280 | |
| 6281 | __m256i iacc_mat_00_5 = _mm256_add_epi16(iacc_mat_00_5_sp1, iacc_mat_00_5_sp2); |
| 6282 | __m256i iacc_mat_01_5 = _mm256_add_epi16(iacc_mat_01_5_sp1, iacc_mat_01_5_sp2); |
| 6283 | __m256i iacc_mat_10_5 = _mm256_add_epi16(iacc_mat_10_5_sp1, iacc_mat_10_5_sp2); |
| 6284 | __m256i iacc_mat_11_5 = _mm256_add_epi16(iacc_mat_11_5_sp1, iacc_mat_11_5_sp2); |
| 6285 | |
| 6286 | __m256i iacc_mat_00_6 = _mm256_add_epi16(iacc_mat_00_6_sp1, iacc_mat_00_6_sp2); |
| 6287 | __m256i iacc_mat_01_6 = _mm256_add_epi16(iacc_mat_01_6_sp1, iacc_mat_01_6_sp2); |
| 6288 | __m256i iacc_mat_10_6 = _mm256_add_epi16(iacc_mat_10_6_sp1, iacc_mat_10_6_sp2); |
| 6289 | __m256i iacc_mat_11_6 = _mm256_add_epi16(iacc_mat_11_6_sp1, iacc_mat_11_6_sp2); |
| 6290 | |
| 6291 | __m256i iacc_mat_00_7 = _mm256_add_epi16(iacc_mat_00_7_sp1, iacc_mat_00_7_sp2); |
| 6292 | __m256i iacc_mat_01_7 = _mm256_add_epi16(iacc_mat_01_7_sp1, iacc_mat_01_7_sp2); |
| 6293 | __m256i iacc_mat_10_7 = _mm256_add_epi16(iacc_mat_10_7_sp1, iacc_mat_10_7_sp2); |
| 6294 | __m256i iacc_mat_11_7 = _mm256_add_epi16(iacc_mat_11_7_sp1, iacc_mat_11_7_sp2); |
| 6295 | |
| 6296 | // Output of both shuffle patterns are added in order to sum dot product outputs of all 32 values in block |
| 6297 | iacc_mat_00_0 = _mm256_madd_epi16(iacc_mat_00_0, scale_0145_0); |
| 6298 | iacc_mat_01_0 = _mm256_madd_epi16(iacc_mat_01_0, scale_2367_0); |
| 6299 | iacc_mat_10_0 = _mm256_madd_epi16(iacc_mat_10_0, scale_0145_0); |
| 6300 | iacc_mat_11_0 = _mm256_madd_epi16(iacc_mat_11_0, scale_2367_0); |
| 6301 | |
| 6302 | iacc_mat_00_1 = _mm256_madd_epi16(iacc_mat_00_1, scale_0145_1); |
| 6303 | iacc_mat_01_1 = _mm256_madd_epi16(iacc_mat_01_1, scale_2367_1); |
| 6304 | iacc_mat_10_1 = _mm256_madd_epi16(iacc_mat_10_1, scale_0145_1); |
| 6305 | iacc_mat_11_1 = _mm256_madd_epi16(iacc_mat_11_1, scale_2367_1); |
| 6306 | |
| 6307 | iacc_mat_00_2 = _mm256_madd_epi16(iacc_mat_00_2, scale_0145_2); |
| 6308 | iacc_mat_01_2 = _mm256_madd_epi16(iacc_mat_01_2, scale_2367_2); |
| 6309 | iacc_mat_10_2 = _mm256_madd_epi16(iacc_mat_10_2, scale_0145_2); |
| 6310 | iacc_mat_11_2 = _mm256_madd_epi16(iacc_mat_11_2, scale_2367_2); |
| 6311 | |
| 6312 | iacc_mat_00_3 = _mm256_madd_epi16(iacc_mat_00_3, scale_0145_3); |
| 6313 | iacc_mat_01_3 = _mm256_madd_epi16(iacc_mat_01_3, scale_2367_3); |
| 6314 | iacc_mat_10_3 = _mm256_madd_epi16(iacc_mat_10_3, scale_0145_3); |
| 6315 | iacc_mat_11_3 = _mm256_madd_epi16(iacc_mat_11_3, scale_2367_3); |
| 6316 | |
| 6317 | iacc_mat_00_4 = _mm256_madd_epi16(iacc_mat_00_4, scale_0145_4); |
| 6318 | iacc_mat_01_4 = _mm256_madd_epi16(iacc_mat_01_4, scale_2367_4); |
| 6319 | iacc_mat_10_4 = _mm256_madd_epi16(iacc_mat_10_4, scale_0145_4); |
| 6320 | iacc_mat_11_4 = _mm256_madd_epi16(iacc_mat_11_4, scale_2367_4); |
| 6321 | |
| 6322 | iacc_mat_00_5 = _mm256_madd_epi16(iacc_mat_00_5, scale_0145_5); |
| 6323 | iacc_mat_01_5 = _mm256_madd_epi16(iacc_mat_01_5, scale_2367_5); |
| 6324 | iacc_mat_10_5 = _mm256_madd_epi16(iacc_mat_10_5, scale_0145_5); |
| 6325 | iacc_mat_11_5 = _mm256_madd_epi16(iacc_mat_11_5, scale_2367_5); |
| 6326 | |
| 6327 | iacc_mat_00_6 = _mm256_madd_epi16(iacc_mat_00_6, scale_0145_6); |
| 6328 | iacc_mat_01_6 = _mm256_madd_epi16(iacc_mat_01_6, scale_2367_6); |
| 6329 | iacc_mat_10_6 = _mm256_madd_epi16(iacc_mat_10_6, scale_0145_6); |
| 6330 | iacc_mat_11_6 = _mm256_madd_epi16(iacc_mat_11_6, scale_2367_6); |
| 6331 | |
| 6332 | iacc_mat_00_7 = _mm256_madd_epi16(iacc_mat_00_7, scale_0145_7); |
| 6333 | iacc_mat_01_7 = _mm256_madd_epi16(iacc_mat_01_7, scale_2367_7); |
| 6334 | iacc_mat_10_7 = _mm256_madd_epi16(iacc_mat_10_7, scale_0145_7); |
| 6335 | iacc_mat_11_7 = _mm256_madd_epi16(iacc_mat_11_7, scale_2367_7); |
| 6336 | |
| 6337 | __m256i iacc_mat_00 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_0, iacc_mat_00_1), _mm256_add_epi32(iacc_mat_00_2, iacc_mat_00_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_00_4, iacc_mat_00_5), _mm256_add_epi32(iacc_mat_00_6, iacc_mat_00_7))); |
| 6338 | __m256i iacc_mat_01 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_0, iacc_mat_01_1), _mm256_add_epi32(iacc_mat_01_2, iacc_mat_01_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_01_4, iacc_mat_01_5), _mm256_add_epi32(iacc_mat_01_6, iacc_mat_01_7))); |
| 6339 | __m256i iacc_mat_10 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_0, iacc_mat_10_1), _mm256_add_epi32(iacc_mat_10_2, iacc_mat_10_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_10_4, iacc_mat_10_5), _mm256_add_epi32(iacc_mat_10_6, iacc_mat_10_7))); |
| 6340 | __m256i iacc_mat_11 = _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_0, iacc_mat_11_1), _mm256_add_epi32(iacc_mat_11_2, iacc_mat_11_3)), _mm256_add_epi32(_mm256_add_epi32(iacc_mat_11_4, iacc_mat_11_5), _mm256_add_epi32(iacc_mat_11_6, iacc_mat_11_7))); |
| 6341 | |
| 6342 | // Straighten out to make 4 row vectors |
| 6343 | __m256i iacc_row_0 = _mm256_blend_epi32(iacc_mat_00, _mm256_shuffle_epi32(iacc_mat_01, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_00 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_01), (int)(78)))), (int)(204))); |
| 6344 | __m256i iacc_row_1 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_00, 78), iacc_mat_01, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_00), (int )(78)))), (__v8si)(__m256i)(iacc_mat_01), (int)(204))); |
| 6345 | __m256i iacc_row_2 = _mm256_blend_epi32(iacc_mat_10, _mm256_shuffle_epi32(iacc_mat_11, 78), 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(iacc_mat_10 ), (__v8si)(__m256i)(((__m256i)__builtin_ia32_pshufd256((__v8si )(__m256i)(iacc_mat_11), (int)(78)))), (int)(204))); |
| 6346 | __m256i iacc_row_3 = _mm256_blend_epi32(_mm256_shuffle_epi32(iacc_mat_10, 78), iacc_mat_11, 204)((__m256i)__builtin_ia32_pblendd256((__v8si)(__m256i)(((__m256i )__builtin_ia32_pshufd256((__v8si)(__m256i)(iacc_mat_10), (int )(78)))), (__v8si)(__m256i)(iacc_mat_11), (int)(204))); |
| 6347 | |
| 6348 | // Load the scale(d) values for all the 4 Q8_k blocks and repeat it across lanes |
| 6349 | const __m128 row_scale_f32_sse = _mm_load_ps(a_ptr[b].d); |
| 6350 | const __m256 row_scale_f32 = _mm256_set_m128(row_scale_f32_sse, row_scale_f32_sse); |
| 6351 | |
| 6352 | // Multiply with appropriate scales and accumulate (for both d and dmin) below |
| 6353 | acc_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_0), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_rows[0]); |
| 6354 | acc_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_1), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_rows[1]); |
| 6355 | acc_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_2), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_rows[2]); |
| 6356 | acc_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_3), _mm256_mul_ps(col_scale_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_rows[3]); |
| 6357 | |
| 6358 | __m256i lhs_bsums_01_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_0123), lhs_raw_bsums_01_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_0123)), (__v2di)(__m128i)(lhs_raw_bsums_01_0123 ), (int)(1))); |
| 6359 | __m256i lhs_bsums_23_0123 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_0123), lhs_raw_bsums_23_0123, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_0123)), (__v2di)(__m128i)(lhs_raw_bsums_23_0123 ), (int)(1))); |
| 6360 | __m256i lhs_bsums_01_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_01_4567), lhs_raw_bsums_01_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_01_4567)), (__v2di)(__m128i)(lhs_raw_bsums_01_4567 ), (int)(1))); |
| 6361 | __m256i lhs_bsums_23_4567 = _mm256_inserti128_si256(_mm256_castsi128_si256(lhs_raw_bsums_23_4567), lhs_raw_bsums_23_4567, 1)((__m256i)__builtin_ia32_insert128i256((__v4di)(__m256i)(_mm256_castsi128_si256 (lhs_raw_bsums_23_4567)), (__v2di)(__m128i)(lhs_raw_bsums_23_4567 ), (int)(1))); |
| 6362 | |
| 6363 | // Take two bsums from two Q8_Ks at a time and multiply with corresponding mins values from each Q2_K |
| 6364 | __m256i iacc_row_min_0_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(0))), mins_01); |
| 6365 | __m256i iacc_row_min_1_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(170))), mins_01); |
| 6366 | __m256i iacc_row_min_2_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(0))), mins_01); |
| 6367 | __m256i iacc_row_min_3_01 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(170))), mins_01); |
| 6368 | |
| 6369 | __m256i iacc_row_min_0_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(85))), mins_23); |
| 6370 | __m256i iacc_row_min_1_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_0123, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_0123 ), (int)(255))), mins_23); |
| 6371 | __m256i iacc_row_min_2_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(85))), mins_23); |
| 6372 | __m256i iacc_row_min_3_23 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_0123, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_0123 ), (int)(255))), mins_23); |
| 6373 | |
| 6374 | __m256i iacc_row_min_0_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(0))), mins_45); |
| 6375 | __m256i iacc_row_min_1_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(170))), mins_45); |
| 6376 | __m256i iacc_row_min_2_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 0)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(0))), mins_45); |
| 6377 | __m256i iacc_row_min_3_45 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 170)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(170))), mins_45); |
| 6378 | |
| 6379 | __m256i iacc_row_min_0_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(85))), mins_67); |
| 6380 | __m256i iacc_row_min_1_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_01_4567, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_01_4567 ), (int)(255))), mins_67); |
| 6381 | __m256i iacc_row_min_2_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 85)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(85))), mins_67); |
| 6382 | __m256i iacc_row_min_3_67 = _mm256_madd_epi16(_mm256_shuffle_epi32(lhs_bsums_23_4567, 255)((__m256i)__builtin_ia32_pshufd256((__v8si)(__m256i)(lhs_bsums_23_4567 ), (int)(255))), mins_67); |
| 6383 | |
| 6384 | __m256i iacc_row_min_0 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_0_01, iacc_row_min_0_23), _mm256_add_epi32(iacc_row_min_0_45,iacc_row_min_0_67)); |
| 6385 | __m256i iacc_row_min_1 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_1_01, iacc_row_min_1_23), _mm256_add_epi32(iacc_row_min_1_45,iacc_row_min_1_67)); |
| 6386 | __m256i iacc_row_min_2 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_2_01, iacc_row_min_2_23), _mm256_add_epi32(iacc_row_min_2_45,iacc_row_min_2_67)); |
| 6387 | __m256i iacc_row_min_3 = _mm256_add_epi32(_mm256_add_epi32(iacc_row_min_3_01, iacc_row_min_3_23), _mm256_add_epi32(iacc_row_min_3_45,iacc_row_min_3_67)); |
| 6388 | |
| 6389 | acc_min_rows[0] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_0), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 0)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(0)))), acc_min_rows[0]); |
| 6390 | acc_min_rows[1] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_1), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 85)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(85)))), acc_min_rows[1]); |
| 6391 | acc_min_rows[2] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_2), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 170)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(170)))), acc_min_rows[2]); |
| 6392 | acc_min_rows[3] = _mm256_fmadd_ps(_mm256_cvtepi32_ps(iacc_row_min_3), _mm256_mul_ps(col_dmin_f32, _mm256_shuffle_ps(row_scale_f32, row_scale_f32, 255)((__m256)__builtin_ia32_shufps256((__v8sf)(__m256)(row_scale_f32 ), (__v8sf)(__m256)(row_scale_f32), (int)(255)))), acc_min_rows[3]); |
| 6393 | } |
| 6394 | } |
| 6395 | // Store the accumulated values |
| 6396 | for (int i = 0; i < 4; i++) { |
| 6397 | _mm256_storeu_ps((float * )(s + ((y * 4 + i) * bs + x * 8)), _mm256_sub_ps(acc_rows[i], acc_min_rows[i])); |
| 6398 | } |
| 6399 | } |
| 6400 | } |
| 6401 | #else |
| 6402 | |
| 6403 | ggml_gemm_q2_K_8x8_q8_K_generic(n, s, bs, vx, vy, nr, nc); |
| 6404 | |
| 6405 | |
| 6406 | #endif |
| 6407 | } |