| File: | root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c |
| Warning: | line 3528, column 25 Value stored to 'grid_index' is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | #define GGML_COMMON_IMPL_C |
| 2 | #include "ggml-common.h" |
| 3 | |
| 4 | #include "ggml-quants.h" |
| 5 | #include "ggml-impl.h" |
| 6 | #include "ggml-cpu/ggml-cpu-impl.h" |
| 7 | #include "ggml-cpu.h" |
| 8 | |
| 9 | #include <math.h> |
| 10 | #include <string.h> |
| 11 | #include <assert.h> |
| 12 | #include <float.h> |
| 13 | #include <stdlib.h> // for qsort |
| 14 | #include <stdio.h> // for GGML_ASSERT |
| 15 | |
| 16 | #ifdef GGML_USE_OPENMP |
| 17 | #include <omp.h> |
| 18 | #endif |
| 19 | |
| 20 | #define GROUP_MAX_EPS1e-15f 1e-15f |
| 21 | #define GROUP_MAX_EPS_IQ3_XXS1e-8f 1e-8f |
| 22 | #define GROUP_MAX_EPS_IQ2_S1e-8f 1e-8f |
| 23 | #define GROUP_MAX_EPS_IQ1_M1e-7f 1e-7f |
| 24 | #define GROUP_MAX_EPS_IQ1_S1e-12f 1e-12f |
| 25 | |
| 26 | #define UNUSEDGGML_UNUSED GGML_UNUSED |
| 27 | |
| 28 | static inline int best_index_int8(int n, const int8_t * val, float x) { |
| 29 | if (x <= val[0]) return 0; |
| 30 | if (x >= val[n-1]) return n-1; |
| 31 | int ml = 0, mu = n-1; |
| 32 | while (mu-ml > 1) { |
| 33 | int mav = (ml+mu)/2; |
| 34 | if (x < val[mav]) mu = mav; else ml = mav; |
| 35 | } |
| 36 | return x - val[mu-1] < val[mu] - x ? mu-1 : mu; |
| 37 | } |
| 38 | |
| 39 | // reference implementation for deterministic creation of model files |
| 40 | void quantize_row_q1_0_ref(const float * GGML_RESTRICTrestrict x, block_q1_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 41 | static const int qk = QK1_0128; |
| 42 | |
| 43 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 43, __extension__ __PRETTY_FUNCTION__); })); |
| 44 | |
| 45 | const int nb = k / qk; |
| 46 | |
| 47 | for (int i = 0; i < nb; i++) { |
| 48 | float sum_abs = 0.0f; |
| 49 | for (int j = 0; j < qk; j++) { |
| 50 | sum_abs += fabsf(x[i*qk + j]); |
| 51 | } |
| 52 | const float d = sum_abs / qk; |
| 53 | |
| 54 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 55 | |
| 56 | // Clear all bits first |
| 57 | for (int j = 0; j < qk / 8; ++j) { |
| 58 | y[i].qs[j] = 0; |
| 59 | } |
| 60 | |
| 61 | // Just store sign of each weight directly (no normalization) |
| 62 | for (int j = 0; j < qk; ++j) { |
| 63 | const int bit_index = j; |
| 64 | const int byte_index = bit_index / 8; |
| 65 | const int bit_offset = bit_index % 8; |
| 66 | |
| 67 | if (x[i*qk + j] >= 0.0f) { |
| 68 | y[i].qs[byte_index] |= (1 << bit_offset); |
| 69 | } |
| 70 | } |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | // reference implementation for deterministic creation of model files |
| 75 | void quantize_row_q4_0_ref(const float * GGML_RESTRICTrestrict x, block_q4_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 76 | static const int qk = QK4_032; |
| 77 | |
| 78 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 78, __extension__ __PRETTY_FUNCTION__); })); |
| 79 | |
| 80 | const int nb = k / qk; |
| 81 | |
| 82 | for (int i = 0; i < nb; i++) { |
| 83 | float amax = 0.0f; // absolute max |
| 84 | float max = 0.0f; |
| 85 | |
| 86 | for (int j = 0; j < qk; j++) { |
| 87 | const float v = x[i*qk + j]; |
| 88 | if (amax < fabsf(v)) { |
| 89 | amax = fabsf(v); |
| 90 | max = v; |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | const float d = max / -8; |
| 95 | const float id = d ? 1.0f/d : 0.0f; |
| 96 | |
| 97 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 98 | |
| 99 | for (int j = 0; j < qk/2; ++j) { |
| 100 | const float x0 = x[i*qk + 0 + j]*id; |
| 101 | const float x1 = x[i*qk + qk/2 + j]*id; |
| 102 | |
| 103 | const uint8_t xi0 = MIN(15, (int8_t)(x0 + 8.5f))((15) < ((int8_t)(x0 + 8.5f)) ? (15) : ((int8_t)(x0 + 8.5f ))); |
| 104 | const uint8_t xi1 = MIN(15, (int8_t)(x1 + 8.5f))((15) < ((int8_t)(x1 + 8.5f)) ? (15) : ((int8_t)(x1 + 8.5f ))); |
| 105 | |
| 106 | y[i].qs[j] = xi0; |
| 107 | y[i].qs[j] |= xi1 << 4; |
| 108 | } |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | void quantize_row_q4_1_ref(const float * GGML_RESTRICTrestrict x, block_q4_1 * GGML_RESTRICTrestrict y, int64_t k) { |
| 113 | const int qk = QK4_132; |
| 114 | |
| 115 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 115, __extension__ __PRETTY_FUNCTION__); })); |
| 116 | |
| 117 | const int nb = k / qk; |
| 118 | |
| 119 | for (int i = 0; i < nb; i++) { |
| 120 | float min = FLT_MAX3.40282347e+38F; |
| 121 | float max = -FLT_MAX3.40282347e+38F; |
| 122 | |
| 123 | for (int j = 0; j < qk; j++) { |
| 124 | const float v = x[i*qk + j]; |
| 125 | |
| 126 | if (v < min) min = v; |
| 127 | if (v > max) max = v; |
| 128 | } |
| 129 | |
| 130 | const float d = (max - min) / ((1 << 4) - 1); |
| 131 | const float id = d ? 1.0f/d : 0.0f; |
| 132 | |
| 133 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 134 | y[i].m = GGML_FP32_TO_FP16(min)ggml_compute_fp32_to_fp16(min); |
| 135 | |
| 136 | for (int j = 0; j < qk/2; ++j) { |
| 137 | const float x0 = (x[i*qk + 0 + j] - min)*id; |
| 138 | const float x1 = (x[i*qk + qk/2 + j] - min)*id; |
| 139 | |
| 140 | const uint8_t xi0 = MIN(15, (int8_t)(x0 + 0.5f))((15) < ((int8_t)(x0 + 0.5f)) ? (15) : ((int8_t)(x0 + 0.5f ))); |
| 141 | const uint8_t xi1 = MIN(15, (int8_t)(x1 + 0.5f))((15) < ((int8_t)(x1 + 0.5f)) ? (15) : ((int8_t)(x1 + 0.5f ))); |
| 142 | |
| 143 | y[i].qs[j] = xi0; |
| 144 | y[i].qs[j] |= xi1 << 4; |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | void quantize_row_q5_0_ref(const float * GGML_RESTRICTrestrict x, block_q5_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 150 | static const int qk = QK5_032; |
| 151 | |
| 152 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 152, __extension__ __PRETTY_FUNCTION__); })); |
| 153 | |
| 154 | const int nb = k / qk; |
| 155 | |
| 156 | for (int i = 0; i < nb; i++) { |
| 157 | float amax = 0.0f; // absolute max |
| 158 | float max = 0.0f; |
| 159 | |
| 160 | for (int j = 0; j < qk; j++) { |
| 161 | const float v = x[i*qk + j]; |
| 162 | if (amax < fabsf(v)) { |
| 163 | amax = fabsf(v); |
| 164 | max = v; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | const float d = max / -16; |
| 169 | const float id = d ? 1.0f/d : 0.0f; |
| 170 | |
| 171 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 172 | |
| 173 | uint32_t qh = 0; |
| 174 | |
| 175 | for (int j = 0; j < qk/2; ++j) { |
| 176 | const float x0 = x[i*qk + 0 + j]*id; |
| 177 | const float x1 = x[i*qk + qk/2 + j]*id; |
| 178 | |
| 179 | const uint8_t xi0 = MIN(31, (int8_t)(x0 + 16.5f))((31) < ((int8_t)(x0 + 16.5f)) ? (31) : ((int8_t)(x0 + 16.5f ))); |
| 180 | const uint8_t xi1 = MIN(31, (int8_t)(x1 + 16.5f))((31) < ((int8_t)(x1 + 16.5f)) ? (31) : ((int8_t)(x1 + 16.5f ))); |
| 181 | |
| 182 | y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); |
| 183 | |
| 184 | // get the 5-th bit and store it in qh at the right position |
| 185 | qh |= ((xi0 & 0x10u) >> 4) << (j + 0); |
| 186 | qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2); |
| 187 | } |
| 188 | |
| 189 | memcpy(&y[i].qh, &qh, sizeof(qh)); |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | void quantize_row_q5_1_ref(const float * GGML_RESTRICTrestrict x, block_q5_1 * GGML_RESTRICTrestrict y, int64_t k) { |
| 194 | const int qk = QK5_132; |
| 195 | |
| 196 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 196, __extension__ __PRETTY_FUNCTION__); })); |
| 197 | |
| 198 | const int nb = k / qk; |
| 199 | |
| 200 | for (int i = 0; i < nb; i++) { |
| 201 | float min = FLT_MAX3.40282347e+38F; |
| 202 | float max = -FLT_MAX3.40282347e+38F; |
| 203 | |
| 204 | for (int j = 0; j < qk; j++) { |
| 205 | const float v = x[i*qk + j]; |
| 206 | |
| 207 | if (v < min) min = v; |
| 208 | if (v > max) max = v; |
| 209 | } |
| 210 | |
| 211 | const float d = (max - min) / ((1 << 5) - 1); |
| 212 | const float id = d ? 1.0f/d : 0.0f; |
| 213 | |
| 214 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 215 | y[i].m = GGML_FP32_TO_FP16(min)ggml_compute_fp32_to_fp16(min); |
| 216 | |
| 217 | uint32_t qh = 0; |
| 218 | |
| 219 | for (int j = 0; j < qk/2; ++j) { |
| 220 | const float x0 = (x[i*qk + 0 + j] - min)*id; |
| 221 | const float x1 = (x[i*qk + qk/2 + j] - min)*id; |
| 222 | |
| 223 | const uint8_t xi0 = (uint8_t)(x0 + 0.5f); |
| 224 | const uint8_t xi1 = (uint8_t)(x1 + 0.5f); |
| 225 | |
| 226 | y[i].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); |
| 227 | |
| 228 | // get the 5-th bit and store it in qh at the right position |
| 229 | qh |= ((xi0 & 0x10u) >> 4) << (j + 0); |
| 230 | qh |= ((xi1 & 0x10u) >> 4) << (j + qk/2); |
| 231 | } |
| 232 | |
| 233 | memcpy(&y[i].qh, &qh, sizeof(y[i].qh)); |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | // reference implementation for deterministic creation of model files |
| 238 | void quantize_row_q8_0_ref(const float * GGML_RESTRICTrestrict x, block_q8_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 239 | assert(k % QK8_0 == 0)((void) sizeof (__assert_single_arg (k % 32 == 0)), __extension__ ({ if (k % 32 == 0) ; else __assert_fail ("k % QK8_0 == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 239, __extension__ __PRETTY_FUNCTION__); })); |
| 240 | const int nb = k / QK8_032; |
| 241 | |
| 242 | for (int i = 0; i < nb; i++) { |
| 243 | float amax = 0.0f; // absolute max |
| 244 | |
| 245 | for (int j = 0; j < QK8_032; j++) { |
| 246 | const float v = x[i*QK8_032 + j]; |
| 247 | amax = MAX(amax, fabsf(v))((amax) > (fabsf(v)) ? (amax) : (fabsf(v))); |
| 248 | } |
| 249 | |
| 250 | const float d = amax / ((1 << 7) - 1); |
| 251 | const float id = d ? 1.0f/d : 0.0f; |
| 252 | |
| 253 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 254 | |
| 255 | for (int j = 0; j < QK8_032; ++j) { |
| 256 | const float x0 = x[i*QK8_032 + j]*id; |
| 257 | |
| 258 | y[i].qs[j] = roundf(x0); |
| 259 | } |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // reference implementation for deterministic creation of model files |
| 264 | void quantize_row_q8_1_ref(const float * GGML_RESTRICTrestrict x, block_q8_1 * GGML_RESTRICTrestrict y, int64_t k) { |
| 265 | assert(QK8_1 == 32)((void) sizeof (__assert_single_arg (32 == 32)), __extension__ ({ if (32 == 32) ; else __assert_fail ("QK8_1 == 32", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 265, __extension__ __PRETTY_FUNCTION__); })); |
| 266 | assert(k % QK8_1 == 0)((void) sizeof (__assert_single_arg (k % 32 == 0)), __extension__ ({ if (k % 32 == 0) ; else __assert_fail ("k % QK8_1 == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 266, __extension__ __PRETTY_FUNCTION__); })); |
| 267 | const int nb = k / QK8_132; |
| 268 | |
| 269 | for (int i = 0; i < nb; i++) { |
| 270 | float amax = 0.0f; // absolute max |
| 271 | |
| 272 | for (int j = 0; j < QK8_132; j++) { |
| 273 | const float v = x[i*QK8_132 + j]; |
| 274 | amax = MAX(amax, fabsf(v))((amax) > (fabsf(v)) ? (amax) : (fabsf(v))); |
| 275 | } |
| 276 | |
| 277 | const float d = amax / ((1 << 7) - 1); |
| 278 | const float id = d ? 1.0f/d : 0.0f; |
| 279 | |
| 280 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 281 | |
| 282 | int sum = 0; |
| 283 | |
| 284 | for (int j = 0; j < QK8_132/2; ++j) { |
| 285 | const float v0 = x[i*QK8_132 + j]*id; |
| 286 | const float v1 = x[i*QK8_132 + QK8_132/2 + j]*id; |
| 287 | |
| 288 | y[i].qs[ j] = roundf(v0); |
| 289 | y[i].qs[QK8_132/2 + j] = roundf(v1); |
| 290 | |
| 291 | sum += y[i].qs[ j]; |
| 292 | sum += y[i].qs[QK8_132/2 + j]; |
| 293 | } |
| 294 | |
| 295 | y[i].s = GGML_FP32_TO_FP16(sum*d)ggml_compute_fp32_to_fp16(sum*d); |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | static inline int best_index_mxfp4(float x, float e) { |
| 300 | int best_index = 0; |
| 301 | float best_err = fabsf(kvalues_mxfp4[0]*e - x); |
| 302 | for (int i = 1; i < 16; i++) { |
| 303 | float err = fabsf(kvalues_mxfp4[i]*e - x); |
| 304 | if (err < best_err) { |
| 305 | best_index = i; |
| 306 | best_err = err; |
| 307 | } |
| 308 | } |
| 309 | return best_index; |
| 310 | } |
| 311 | |
| 312 | void quantize_row_mxfp4_ref(const float * GGML_RESTRICTrestrict x, block_mxfp4 * GGML_RESTRICTrestrict y, int64_t k) { |
| 313 | static const int qk = QK_MXFP432; |
| 314 | |
| 315 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 315, __extension__ __PRETTY_FUNCTION__); })); |
| 316 | |
| 317 | const int nb = k / qk; |
| 318 | |
| 319 | for (int i = 0; i < nb; i++) { |
| 320 | float amax = 0.0f; // absolute max |
| 321 | |
| 322 | for (int j = 0; j < qk; j++) { |
| 323 | const float v = x[i*qk + j]; |
| 324 | |
| 325 | if (amax < fabsf(v)) { |
| 326 | amax = fabsf(v); |
| 327 | } |
| 328 | } |
| 329 | |
| 330 | const uint8_t e = amax > 0.0f ? (uint8_t) (floorf(log2f(amax)) - 2 + 127) : 0; |
| 331 | |
| 332 | const float d = GGML_E8M0_TO_FP32_HALF(e)ggml_e8m0_to_fp32_half(e); |
| 333 | |
| 334 | y[i].e = e; |
| 335 | |
| 336 | for (int j = 0; j < qk/2; ++j) { |
| 337 | const uint8_t x0 = best_index_mxfp4(x[i*qk + 0 + j], d); |
| 338 | const uint8_t x1 = best_index_mxfp4(x[i*qk + qk/2 + j], d); |
| 339 | |
| 340 | y[i].qs[j] = x0; |
| 341 | y[i].qs[j] |= x1 << 4; |
| 342 | } |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | void quantize_row_nvfp4_ref(const float * GGML_RESTRICTrestrict x, block_nvfp4 * GGML_RESTRICTrestrict y, int64_t k) { |
| 347 | static const int qk = QK_NVFP464; |
| 348 | static const int qk_sub = QK_NVFP4_SUB16; |
| 349 | static const int n_sub = QK_NVFP464 / QK_NVFP4_SUB16; |
| 350 | |
| 351 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 351, __extension__ __PRETTY_FUNCTION__); })); |
| 352 | |
| 353 | const int nb = k / qk; |
| 354 | |
| 355 | for (int i = 0; i < nb; i++) { |
| 356 | for (int s = 0; s < n_sub; s++) { |
| 357 | const float * xb = x + i*qk + s*qk_sub; |
| 358 | |
| 359 | float amax = 0.0f; |
| 360 | for (int j = 0; j < qk_sub; j++) { |
| 361 | if (amax < fabsf(xb[j])) { |
| 362 | amax = fabsf(xb[j]); |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | // UE4M3 scale: amax / 6.0 maps the max E2M1 value (6.0) to amax |
| 367 | const uint8_t ue = ggml_fp32_to_ue4m3(amax / 6.0f); |
| 368 | y[i].d[s] = ue; |
| 369 | const float d = ggml_ue4m3_to_fp32(ue); |
| 370 | |
| 371 | for (int j = 0; j < qk_sub/2; ++j) { |
| 372 | const uint8_t x0 = best_index_mxfp4(xb[0 + j], d); |
| 373 | const uint8_t x1 = best_index_mxfp4(xb[qk_sub/2 + j], d); |
| 374 | |
| 375 | y[i].qs[s*(qk_sub/2) + j] = x0 | (x1 << 4); |
| 376 | } |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 382 | static const int qk = QK1_0128; |
| 383 | |
| 384 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 384, __extension__ __PRETTY_FUNCTION__); })); |
| 385 | |
| 386 | const int nb = k / qk; |
| 387 | |
| 388 | for (int i = 0; i < nb; i++) { |
| 389 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 390 | const float neg_d = -d; |
| 391 | |
| 392 | for (int j = 0; j < qk; ++j) { |
| 393 | const int byte_index = j / 8; |
| 394 | const int bit_offset = j % 8; |
| 395 | const uint8_t bit = (x[i].qs[byte_index] >> bit_offset) & 1; |
| 396 | y[i*qk + j] = bit ? d : neg_d; |
| 397 | } |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 402 | static const int qk = QK4_032; |
| 403 | |
| 404 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 404, __extension__ __PRETTY_FUNCTION__); })); |
| 405 | |
| 406 | const int nb = k / qk; |
| 407 | |
| 408 | for (int i = 0; i < nb; i++) { |
| 409 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 410 | |
| 411 | for (int j = 0; j < qk/2; ++j) { |
| 412 | const int x0 = (x[i].qs[j] & 0x0F) - 8; |
| 413 | const int x1 = (x[i].qs[j] >> 4) - 8; |
| 414 | |
| 415 | y[i*qk + j + 0 ] = x0*d; |
| 416 | y[i*qk + j + qk/2] = x1*d; |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 422 | static const int qk = QK4_132; |
| 423 | |
| 424 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 424, __extension__ __PRETTY_FUNCTION__); })); |
| 425 | |
| 426 | const int nb = k / qk; |
| 427 | |
| 428 | for (int i = 0; i < nb; i++) { |
| 429 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 430 | const float m = GGML_FP16_TO_FP32(x[i].m)ggml_compute_fp16_to_fp32(x[i].m); |
| 431 | |
| 432 | for (int j = 0; j < qk/2; ++j) { |
| 433 | const int x0 = (x[i].qs[j] & 0x0F); |
| 434 | const int x1 = (x[i].qs[j] >> 4); |
| 435 | |
| 436 | y[i*qk + j + 0 ] = x0*d + m; |
| 437 | y[i*qk + j + qk/2] = x1*d + m; |
| 438 | } |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 443 | static const int qk = QK5_032; |
| 444 | |
| 445 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 445, __extension__ __PRETTY_FUNCTION__); })); |
| 446 | |
| 447 | const int nb = k / qk; |
| 448 | |
| 449 | for (int i = 0; i < nb; i++) { |
| 450 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 451 | |
| 452 | uint32_t qh; |
| 453 | memcpy(&qh, x[i].qh, sizeof(qh)); |
| 454 | |
| 455 | for (int j = 0; j < qk/2; ++j) { |
| 456 | const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10; |
| 457 | const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10; |
| 458 | |
| 459 | const int32_t x0 = ((x[i].qs[j] & 0x0F) | xh_0) - 16; |
| 460 | const int32_t x1 = ((x[i].qs[j] >> 4) | xh_1) - 16; |
| 461 | |
| 462 | y[i*qk + j + 0 ] = x0*d; |
| 463 | y[i*qk + j + qk/2] = x1*d; |
| 464 | } |
| 465 | } |
| 466 | } |
| 467 | |
| 468 | void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 469 | static const int qk = QK5_132; |
| 470 | |
| 471 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 471, __extension__ __PRETTY_FUNCTION__); })); |
| 472 | |
| 473 | const int nb = k / qk; |
| 474 | |
| 475 | for (int i = 0; i < nb; i++) { |
| 476 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 477 | const float m = GGML_FP16_TO_FP32(x[i].m)ggml_compute_fp16_to_fp32(x[i].m); |
| 478 | |
| 479 | uint32_t qh; |
| 480 | memcpy(&qh, x[i].qh, sizeof(qh)); |
| 481 | |
| 482 | for (int j = 0; j < qk/2; ++j) { |
| 483 | const uint8_t xh_0 = ((qh >> (j + 0)) << 4) & 0x10; |
| 484 | const uint8_t xh_1 = ((qh >> (j + 12)) ) & 0x10; |
| 485 | |
| 486 | const int x0 = (x[i].qs[j] & 0x0F) | xh_0; |
| 487 | const int x1 = (x[i].qs[j] >> 4) | xh_1; |
| 488 | |
| 489 | y[i*qk + j + 0 ] = x0*d + m; |
| 490 | y[i*qk + j + qk/2] = x1*d + m; |
| 491 | } |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 496 | static const int qk = QK8_032; |
| 497 | |
| 498 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 498, __extension__ __PRETTY_FUNCTION__); })); |
| 499 | |
| 500 | const int nb = k / qk; |
| 501 | |
| 502 | for (int i = 0; i < nb; i++) { |
| 503 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 504 | |
| 505 | for (int j = 0; j < qk; ++j) { |
| 506 | y[i*qk + j] = x[i].qs[j]*d; |
| 507 | } |
| 508 | } |
| 509 | } |
| 510 | |
| 511 | void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 512 | static const int qk = QK_MXFP432; |
| 513 | |
| 514 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 514, __extension__ __PRETTY_FUNCTION__); })); |
| 515 | |
| 516 | const int nb = k / qk; |
| 517 | |
| 518 | for (int i = 0; i < nb; i++) { |
| 519 | const float d = GGML_E8M0_TO_FP32_HALF(x[i].e)ggml_e8m0_to_fp32_half(x[i].e); |
| 520 | |
| 521 | for (int j = 0; j < qk/2; ++j) { |
| 522 | const int8_t x0 = kvalues_mxfp4[x[i].qs[j] & 0x0F]; |
| 523 | const int8_t x1 = kvalues_mxfp4[x[i].qs[j] >> 4]; |
| 524 | |
| 525 | y[i*qk + j + 0 ] = x0*d; |
| 526 | y[i*qk + j + qk/2] = x1*d; |
| 527 | } |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 532 | static const int qk = QK_NVFP464; |
| 533 | static const int qk_sub = QK_NVFP4_SUB16; |
| 534 | static const int n_sub = QK_NVFP464 / QK_NVFP4_SUB16; |
| 535 | |
| 536 | assert(k % qk == 0)((void) sizeof (__assert_single_arg (k % qk == 0)), __extension__ ({ if (k % qk == 0) ; else __assert_fail ("k % qk == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 536, __extension__ __PRETTY_FUNCTION__); })); |
| 537 | |
| 538 | const int nb = k / qk; |
| 539 | |
| 540 | for (int i = 0; i < nb; i++) { |
| 541 | for (int s = 0; s < n_sub; s++) { |
| 542 | const float d = ggml_ue4m3_to_fp32(x[i].d[s]); |
| 543 | float * yb = y + i*qk + s*qk_sub; |
| 544 | |
| 545 | for (int j = 0; j < qk_sub/2; ++j) { |
| 546 | const int8_t v0 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] & 0x0F]; |
| 547 | const int8_t v1 = kvalues_mxfp4[x[i].qs[s*(qk_sub/2) + j] >> 4]; |
| 548 | |
| 549 | yb[j + 0 ] = v0*d; |
| 550 | yb[j + qk_sub/2] = v1*d; |
| 551 | } |
| 552 | } |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | // |
| 557 | // 2-6 bit quantization in super-blocks |
| 558 | // |
| 559 | |
| 560 | // |
| 561 | // ===================== Helper functions |
| 562 | // |
| 563 | static inline int nearest_int(float fval) { |
| 564 | assert(fabsf(fval) <= 4194303.f)((void) sizeof (__assert_single_arg (fabsf(fval) <= 4194303.f )), __extension__ ({ if (fabsf(fval) <= 4194303.f) ; else __assert_fail ("fabsf(fval) <= 4194303.f", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 564, __extension__ __PRETTY_FUNCTION__); })); |
| 565 | float val = fval + 12582912.f; |
| 566 | int i; memcpy(&i, &val, sizeof(int)); |
| 567 | return (i & 0x007fffff) - 0x00400000; |
| 568 | } |
| 569 | |
| 570 | static float make_qx_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, int8_t * GGML_RESTRICTrestrict L, int rmse_type, |
| 571 | const float * GGML_RESTRICTrestrict qw) { |
| 572 | float max = 0; |
| 573 | float amax = 0; |
| 574 | for (int i = 0; i < n; ++i) { |
| 575 | float ax = fabsf(x[i]); |
| 576 | if (ax > amax) { amax = ax; max = x[i]; } |
| 577 | } |
| 578 | if (amax < GROUP_MAX_EPS1e-15f) { // all zero |
| 579 | for (int i = 0; i < n; ++i) { |
| 580 | L[i] = 0; |
| 581 | } |
| 582 | return 0.f; |
| 583 | } |
| 584 | float iscale = -nmax / max; |
| 585 | if (rmse_type == 0) { |
| 586 | for (int i = 0; i < n; ++i) { |
| 587 | int l = nearest_int(iscale * x[i]); |
| 588 | L[i] = nmax + MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 589 | } |
| 590 | return 1/iscale; |
| 591 | } |
| 592 | bool_Bool return_early = false0; |
| 593 | if (rmse_type < 0) { |
| 594 | rmse_type = -rmse_type; |
| 595 | return_early = true1; |
| 596 | } |
| 597 | float sumlx = 0; |
| 598 | float suml2 = 0; |
| 599 | #ifdef HAVE_BUGGY_APPLE_LINKER |
| 600 | // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7 |
| 601 | for (volatile int i = 0; i < n; ++i) { |
| 602 | #else |
| 603 | for (int i = 0; i < n; ++i) { |
| 604 | #endif |
| 605 | int l = nearest_int(iscale * x[i]); |
| 606 | l = MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 607 | L[i] = l + nmax; |
| 608 | float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i])); |
| 609 | sumlx += w*x[i]*l; |
| 610 | suml2 += w*l*l; |
| 611 | } |
| 612 | float scale = suml2 ? sumlx/suml2 : 0.0f; |
| 613 | if (return_early) return suml2 > 0 ? 0.5f*(scale + 1/iscale) : 1/iscale; |
| 614 | float best = scale * sumlx; |
| 615 | for (int is = -9; is <= 9; ++is) { |
| 616 | if (is == 0) { |
| 617 | continue; |
| 618 | } |
| 619 | iscale = -(nmax + 0.1f*is) / max; |
| 620 | sumlx = suml2 = 0; |
| 621 | for (int i = 0; i < n; ++i) { |
| 622 | int l = nearest_int(iscale * x[i]); |
| 623 | l = MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 624 | float w = qw ? qw[i] : rmse_type == 1 ? x[i] * x[i] : rmse_type == 2 ? 1 : rmse_type == 3 ? fabsf(x[i]) : sqrtf(fabsf(x[i])); |
| 625 | sumlx += w*x[i]*l; |
| 626 | suml2 += w*l*l; |
| 627 | } |
| 628 | if (suml2 > 0 && sumlx*sumlx > best*suml2) { |
| 629 | for (int i = 0; i < n; ++i) { |
| 630 | int l = nearest_int(iscale * x[i]); |
| 631 | L[i] = nmax + MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 632 | } |
| 633 | scale = sumlx/suml2; best = scale*sumlx; |
| 634 | } |
| 635 | } |
| 636 | return scale; |
| 637 | } |
| 638 | |
| 639 | static float make_q3_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, int8_t * GGML_RESTRICTrestrict L, bool_Bool do_rmse) { |
| 640 | float max = 0; |
| 641 | float amax = 0; |
| 642 | for (int i = 0; i < n; ++i) { |
| 643 | float ax = fabsf(x[i]); |
| 644 | if (ax > amax) { amax = ax; max = x[i]; } |
| 645 | } |
| 646 | if (amax < GROUP_MAX_EPS1e-15f) { // all zero |
| 647 | for (int i = 0; i < n; ++i) { L[i] = 0; } |
| 648 | return 0.f; |
| 649 | } |
| 650 | float iscale = -nmax / max; |
| 651 | if (do_rmse) { |
| 652 | float sumlx = 0; |
| 653 | float suml2 = 0; |
| 654 | for (int i = 0; i < n; ++i) { |
| 655 | int l = nearest_int(iscale * x[i]); |
| 656 | l = MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 657 | L[i] = l; |
| 658 | float w = x[i]*x[i]; |
| 659 | sumlx += w*x[i]*l; |
| 660 | suml2 += w*l*l; |
| 661 | } |
| 662 | for (int itry = 0; itry < 5; ++itry) { |
| 663 | int n_changed = 0; |
| 664 | for (int i = 0; i < n; ++i) { |
| 665 | float w = x[i]*x[i]; |
| 666 | float slx = sumlx - w*x[i]*L[i]; |
| 667 | if (slx > 0) { |
| 668 | float sl2 = suml2 - w*L[i]*L[i]; |
| 669 | int new_l = nearest_int(x[i] * sl2 / slx); |
| 670 | new_l = MAX(-nmax, MIN(nmax-1, new_l))((-nmax) > (((nmax-1) < (new_l) ? (nmax-1) : (new_l))) ? (-nmax) : (((nmax-1) < (new_l) ? (nmax-1) : (new_l)))); |
| 671 | if (new_l != L[i]) { |
| 672 | slx += w*x[i]*new_l; |
| 673 | sl2 += w*new_l*new_l; |
| 674 | if (sl2 > 0 && slx*slx*suml2 > sumlx*sumlx*sl2) { |
| 675 | L[i] = new_l; sumlx = slx; suml2 = sl2; |
| 676 | ++n_changed; |
| 677 | } |
| 678 | } |
| 679 | } |
| 680 | } |
| 681 | if (!n_changed) { |
| 682 | break; |
| 683 | } |
| 684 | } |
| 685 | for (int i = 0; i < n; ++i) { |
| 686 | L[i] += nmax; |
| 687 | } |
| 688 | return suml2 > 0.0f ? sumlx / suml2 : 0.0f; |
| 689 | } |
| 690 | for (int i = 0; i < n; ++i) { |
| 691 | int l = nearest_int(iscale * x[i]); |
| 692 | l = MAX(-nmax, MIN(nmax-1, l))((-nmax) > (((nmax-1) < (l) ? (nmax-1) : (l))) ? (-nmax ) : (((nmax-1) < (l) ? (nmax-1) : (l)))); |
| 693 | L[i] = l + nmax; |
| 694 | } |
| 695 | return 1/iscale; |
| 696 | } |
| 697 | |
| 698 | static float make_qkx1_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, uint8_t * GGML_RESTRICTrestrict L, float * GGML_RESTRICTrestrict the_min, |
| 699 | int ntry, float alpha) { |
| 700 | float min = x[0]; |
| 701 | float max = x[0]; |
| 702 | for (int i = 1; i < n; ++i) { |
| 703 | if (x[i] < min) min = x[i]; |
| 704 | if (x[i] > max) max = x[i]; |
| 705 | } |
| 706 | if (max == min) { |
| 707 | for (int i = 0; i < n; ++i) L[i] = 0; |
| 708 | *the_min = 0; |
| 709 | return 0.f; |
| 710 | } |
| 711 | if (min > 0) min = 0; |
| 712 | float iscale = nmax/(max - min); |
| 713 | float scale = 1/iscale; |
| 714 | for (int itry = 0; itry < ntry; ++itry) { |
| 715 | float sumlx = 0; int suml2 = 0; |
| 716 | bool_Bool did_change = false0; |
| 717 | for (int i = 0; i < n; ++i) { |
| 718 | int l = nearest_int(iscale*(x[i] - min)); |
| 719 | l = MAX(0, MIN(nmax, l))((0) > (((nmax) < (l) ? (nmax) : (l))) ? (0) : (((nmax) < (l) ? (nmax) : (l)))); |
| 720 | if (l != L[i]) { |
| 721 | L[i] = l; |
| 722 | did_change = true1; |
| 723 | } |
| 724 | sumlx += (x[i] - min)*l; |
| 725 | suml2 += l*l; |
| 726 | } |
| 727 | scale = sumlx/suml2; |
| 728 | float sum = 0; |
| 729 | for (int i = 0; i < n; ++i) { |
| 730 | sum += x[i] - scale*L[i]; |
| 731 | } |
| 732 | min = alpha*min + (1 - alpha)*sum/n; |
| 733 | if (min > 0) min = 0; |
| 734 | iscale = 1/scale; |
| 735 | if (!did_change) break; |
| 736 | } |
| 737 | *the_min = -min; |
| 738 | return scale; |
| 739 | } |
| 740 | |
| 741 | static float make_qkx2_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, const float * GGML_RESTRICTrestrict weights, |
| 742 | uint8_t * GGML_RESTRICTrestrict L, float * GGML_RESTRICTrestrict the_min, uint8_t * GGML_RESTRICTrestrict Laux, |
| 743 | float rmin, float rdelta, int nstep, bool_Bool use_mad) { |
| 744 | float min = x[0]; |
| 745 | float max = x[0]; |
| 746 | float sum_w = weights[0]; |
| 747 | float sum_x = sum_w * x[0]; |
| 748 | #ifdef HAVE_BUGGY_APPLE_LINKER |
| 749 | // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7 |
| 750 | for (volatile int i = 1; i < n; ++i) { |
| 751 | #else |
| 752 | for (int i = 1; i < n; ++i) { |
| 753 | #endif |
| 754 | if (x[i] < min) min = x[i]; |
| 755 | if (x[i] > max) max = x[i]; |
| 756 | float w = weights[i]; |
| 757 | sum_w += w; |
| 758 | sum_x += w * x[i]; |
| 759 | } |
| 760 | if (min > 0) min = 0; |
| 761 | if (max == min) { |
| 762 | for (int i = 0; i < n; ++i) L[i] = 0; |
| 763 | *the_min = -min; |
| 764 | return 0.f; |
| 765 | } |
| 766 | float iscale = nmax/(max - min); |
| 767 | float scale = 1/iscale; |
| 768 | float best_error = 0; |
| 769 | for (int i = 0; i < n; ++i) { |
| 770 | int l = nearest_int(iscale*(x[i] - min)); |
| 771 | L[i] = MAX(0, MIN(nmax, l))((0) > (((nmax) < (l) ? (nmax) : (l))) ? (0) : (((nmax) < (l) ? (nmax) : (l)))); |
| 772 | float diff = scale * L[i] + min - x[i]; |
| 773 | diff = use_mad ? fabsf(diff) : diff * diff; |
| 774 | float w = weights[i]; |
| 775 | best_error += w * diff; |
| 776 | } |
| 777 | if (nstep < 1) { |
| 778 | *the_min = -min; |
| 779 | return scale; |
| 780 | } |
| 781 | for (int is = 0; is <= nstep; ++is) { |
| 782 | iscale = (rmin + rdelta*is + nmax)/(max - min); |
| 783 | float sum_l = 0, sum_l2 = 0, sum_xl = 0; |
| 784 | for (int i = 0; i < n; ++i) { |
| 785 | int l = nearest_int(iscale*(x[i] - min)); |
| 786 | l = MAX(0, MIN(nmax, l))((0) > (((nmax) < (l) ? (nmax) : (l))) ? (0) : (((nmax) < (l) ? (nmax) : (l)))); |
| 787 | Laux[i] = l; |
| 788 | float w = weights[i]; |
| 789 | sum_l += w*l; |
| 790 | sum_l2 += w*l*l; |
| 791 | sum_xl += w*l*x[i]; |
| 792 | } |
| 793 | float D = sum_w * sum_l2 - sum_l * sum_l; |
| 794 | if (D > 0) { |
| 795 | float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D; |
| 796 | float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D; |
| 797 | if (this_min > 0) { |
| 798 | this_min = 0; |
| 799 | this_scale = sum_xl / sum_l2; |
| 800 | } |
| 801 | float cur_error = 0; |
| 802 | for (int i = 0; i < n; ++i) { |
| 803 | float diff = this_scale * Laux[i] + this_min - x[i]; |
| 804 | diff = use_mad ? fabsf(diff) : diff * diff; |
| 805 | float w = weights[i]; |
| 806 | cur_error += w * diff; |
| 807 | } |
| 808 | if (cur_error < best_error) { |
| 809 | for (int i = 0; i < n; ++i) { |
| 810 | L[i] = Laux[i]; |
| 811 | } |
| 812 | best_error = cur_error; |
| 813 | scale = this_scale; |
| 814 | min = this_min; |
| 815 | } |
| 816 | } |
| 817 | } |
| 818 | *the_min = -min; |
| 819 | return scale; |
| 820 | } |
| 821 | |
| 822 | static inline void get_scale_min_k4(int j, const uint8_t * GGML_RESTRICTrestrict q, uint8_t * GGML_RESTRICTrestrict d, uint8_t * GGML_RESTRICTrestrict m) { |
| 823 | if (j < 4) { |
| 824 | *d = q[j] & 63; *m = q[j + 4] & 63; |
| 825 | } else { |
| 826 | *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); |
| 827 | *m = (q[j+4] >> 4) | ((q[j-0] >> 6) << 4); |
| 828 | } |
| 829 | } |
| 830 | |
| 831 | //========================- 2-bit (de)-quantization |
| 832 | |
| 833 | void quantize_row_q2_K_ref(const float * GGML_RESTRICTrestrict x, block_q2_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 834 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 834, __extension__ __PRETTY_FUNCTION__); })); |
| 835 | const int nb = k / QK_K256; |
| 836 | |
| 837 | uint8_t L[QK_K256]; |
| 838 | uint8_t Laux[16]; |
| 839 | float weights[16]; |
| 840 | float mins[QK_K256/16]; |
| 841 | float scales[QK_K256/16]; |
| 842 | |
| 843 | const float q4scale = 15.f; |
| 844 | |
| 845 | for (int i = 0; i < nb; i++) { |
| 846 | float max_scale = 0; // as we are deducting the min, scales are always positive |
| 847 | float max_min = 0; |
| 848 | for (int j = 0; j < QK_K256/16; ++j) { |
| 849 | for (int l = 0; l < 16; ++l) weights[l] = fabsf(x[16*j + l]); |
| 850 | scales[j] = make_qkx2_quants(16, 3, x + 16*j, weights, L + 16*j, &mins[j], Laux, -0.5f, 0.1f, 15, true1); |
| 851 | float scale = scales[j]; |
| 852 | if (scale > max_scale) { |
| 853 | max_scale = scale; |
| 854 | } |
| 855 | float min = mins[j]; |
| 856 | if (min > max_min) { |
| 857 | max_min = min; |
| 858 | } |
| 859 | } |
| 860 | |
| 861 | if (max_scale > 0) { |
| 862 | float iscale = q4scale/max_scale; |
| 863 | for (int j = 0; j < QK_K256/16; ++j) { |
| 864 | int l = nearest_int(iscale*scales[j]); |
| 865 | y[i].scales[j] = l; |
| 866 | } |
| 867 | y[i].d = GGML_FP32_TO_FP16(max_scale/q4scale)ggml_compute_fp32_to_fp16(max_scale/q4scale); |
| 868 | } else { |
| 869 | for (int j = 0; j < QK_K256/16; ++j) y[i].scales[j] = 0; |
| 870 | y[i].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 871 | } |
| 872 | if (max_min > 0) { |
| 873 | float iscale = q4scale/max_min; |
| 874 | for (int j = 0; j < QK_K256/16; ++j) { |
| 875 | int l = nearest_int(iscale*mins[j]); |
| 876 | y[i].scales[j] |= (l << 4); |
| 877 | } |
| 878 | y[i].dmin = GGML_FP32_TO_FP16(max_min/q4scale)ggml_compute_fp32_to_fp16(max_min/q4scale); |
| 879 | } else { |
| 880 | y[i].dmin = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 881 | } |
| 882 | for (int j = 0; j < QK_K256/16; ++j) { |
| 883 | const float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * (y[i].scales[j] & 0xF); |
| 884 | if (!d) continue; |
| 885 | const float dm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin) * (y[i].scales[j] >> 4); |
| 886 | for (int ii = 0; ii < 16; ++ii) { |
| 887 | int l = nearest_int((x[16*j + ii] + dm)/d); |
| 888 | l = MAX(0, MIN(3, l))((0) > (((3) < (l) ? (3) : (l))) ? (0) : (((3) < (l) ? (3) : (l)))); |
| 889 | L[16*j + ii] = l; |
| 890 | } |
| 891 | } |
| 892 | |
| 893 | for (int j = 0; j < QK_K256; j += 128) { |
| 894 | for (int l = 0; l < 32; ++l) { |
| 895 | y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); |
| 896 | } |
| 897 | } |
| 898 | |
| 899 | x += QK_K256; |
| 900 | } |
| 901 | } |
| 902 | |
| 903 | void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 904 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 904, __extension__ __PRETTY_FUNCTION__); })); |
| 905 | const int nb = k / QK_K256; |
| 906 | |
| 907 | for (int i = 0; i < nb; i++) { |
| 908 | |
| 909 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 910 | const float min = GGML_FP16_TO_FP32(x[i].dmin)ggml_compute_fp16_to_fp32(x[i].dmin); |
| 911 | |
| 912 | const uint8_t * q = x[i].qs; |
| 913 | |
| 914 | int is = 0; |
| 915 | float dl, ml; |
| 916 | for (int n = 0; n < QK_K256; n += 128) { |
| 917 | int shift = 0; |
| 918 | for (int j = 0; j < 4; ++j) { |
| 919 | |
| 920 | uint8_t sc = x[i].scales[is++]; |
| 921 | dl = d * (sc & 0xF); ml = min * (sc >> 4); |
| 922 | for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l] >> shift) & 3)) - ml; |
| 923 | |
| 924 | sc = x[i].scales[is++]; |
| 925 | dl = d * (sc & 0xF); ml = min * (sc >> 4); |
| 926 | for (int l = 0; l < 16; ++l) *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3)) - ml; |
| 927 | |
| 928 | shift += 2; |
| 929 | } |
| 930 | q += 32; |
| 931 | } |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | static float make_qkx3_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, const float * GGML_RESTRICTrestrict weights, |
| 936 | uint8_t * GGML_RESTRICTrestrict L, float * GGML_RESTRICTrestrict the_min, uint8_t * GGML_RESTRICTrestrict Laux, |
| 937 | float rmin, float rdelta, int nstep, bool_Bool use_mad) { |
| 938 | float min = x[0]; |
| 939 | float max = x[0]; |
| 940 | float sum_w = weights ? weights[0] : x[0]*x[0]; |
| 941 | float sum_x = sum_w * x[0]; |
| 942 | #ifdef HAVE_BUGGY_APPLE_LINKER |
| 943 | // use 'volatile' to prevent unroll and work around a bug in Apple ld64 1015.7 |
| 944 | for (volatile int i = 1; i < n; ++i) { |
| 945 | #else |
| 946 | for (int i = 1; i < n; ++i) { |
| 947 | #endif |
| 948 | if (x[i] < min) min = x[i]; |
| 949 | if (x[i] > max) max = x[i]; |
| 950 | float w = weights ? weights[i] : x[i]*x[i]; |
| 951 | sum_w += w; |
| 952 | sum_x += w * x[i]; |
| 953 | } |
| 954 | if (min > 0) { |
| 955 | min = 0; |
| 956 | } |
| 957 | if (max <= min) { |
| 958 | memset(L, 0, n); |
| 959 | *the_min = -min; |
| 960 | return 0.f; |
| 961 | } |
| 962 | float iscale = nmax/(max - min); |
| 963 | float scale = 1/iscale; |
| 964 | float best_mad = 0; |
| 965 | for (int i = 0; i < n; ++i) { |
| 966 | int l = nearest_int(iscale*(x[i] - min)); |
| 967 | L[i] = MAX(0, MIN(nmax, l))((0) > (((nmax) < (l) ? (nmax) : (l))) ? (0) : (((nmax) < (l) ? (nmax) : (l)))); |
| 968 | float diff = scale * L[i] + min - x[i]; |
| 969 | diff = use_mad ? fabsf(diff) : diff*diff; |
| 970 | float w = weights ? weights[i] : x[i]*x[i]; |
| 971 | best_mad += w * diff; |
| 972 | } |
| 973 | if (nstep < 1) { |
| 974 | *the_min = -min; |
| 975 | return scale; |
| 976 | } |
| 977 | for (int is = 0; is <= nstep; ++is) { |
| 978 | iscale = (rmin + rdelta*is + nmax)/(max - min); |
| 979 | float sum_l = 0, sum_l2 = 0, sum_xl = 0; |
| 980 | for (int i = 0; i < n; ++i) { |
| 981 | int l = nearest_int(iscale*(x[i] - min)); |
| 982 | l = MAX(0, MIN(nmax, l))((0) > (((nmax) < (l) ? (nmax) : (l))) ? (0) : (((nmax) < (l) ? (nmax) : (l)))); |
| 983 | Laux[i] = l; |
| 984 | float w = weights ? weights[i] : x[i]*x[i]; |
| 985 | sum_l += w*l; |
| 986 | sum_l2 += w*l*l; |
| 987 | sum_xl += w*l*x[i]; |
| 988 | } |
| 989 | float D = sum_w * sum_l2 - sum_l * sum_l; |
| 990 | if (D > 0) { |
| 991 | float this_scale = (sum_w * sum_xl - sum_x * sum_l)/D; |
| 992 | float this_min = (sum_l2 * sum_x - sum_l * sum_xl)/D; |
| 993 | if (this_min > 0) { |
| 994 | this_min = 0; |
| 995 | this_scale = sum_xl / sum_l2; |
| 996 | } |
| 997 | float mad = 0; |
| 998 | for (int i = 0; i < n; ++i) { |
| 999 | float diff = this_scale * Laux[i] + this_min - x[i]; |
| 1000 | diff = use_mad ? fabsf(diff) : diff*diff; |
| 1001 | float w = weights ? weights[i] : x[i]*x[i]; |
| 1002 | mad += w * diff; |
| 1003 | } |
| 1004 | if (mad < best_mad) { |
| 1005 | for (int i = 0; i < n; ++i) { |
| 1006 | L[i] = Laux[i]; |
| 1007 | } |
| 1008 | best_mad = mad; |
| 1009 | scale = this_scale; |
| 1010 | min = this_min; |
| 1011 | } |
| 1012 | } |
| 1013 | } |
| 1014 | *the_min = -min; |
| 1015 | return scale; |
| 1016 | } |
| 1017 | |
| 1018 | static float make_qp_quants(int n, int nmax, const float * GGML_RESTRICTrestrict x, uint8_t * GGML_RESTRICTrestrict L, const float * quant_weights) { |
| 1019 | float max = 0; |
| 1020 | for (int i = 0; i < n; ++i) { |
| 1021 | max = MAX(max, x[i])((max) > (x[i]) ? (max) : (x[i])); |
| 1022 | } |
| 1023 | if (max < GROUP_MAX_EPS1e-15f) { // all zero |
| 1024 | for (int i = 0; i < n; ++i) { L[i] = 0; } |
| 1025 | return 0.f; |
| 1026 | } |
| 1027 | float iscale = nmax / max; |
| 1028 | for (int i = 0; i < n; ++i) { |
| 1029 | L[i] = nearest_int(iscale * x[i]); |
| 1030 | } |
| 1031 | float scale = 1/iscale; |
| 1032 | float best_mse = 0; |
| 1033 | for (int i = 0; i < n; ++i) { |
| 1034 | float diff = x[i] - scale*L[i]; |
| 1035 | float w = quant_weights[i]; |
| 1036 | best_mse += w*diff*diff; |
| 1037 | } |
| 1038 | for (int is = -4; is <= 4; ++is) { |
| 1039 | if (is == 0) continue; |
| 1040 | float iscale_is = (0.1f*is + nmax)/max; |
| 1041 | float scale_is = 1/iscale_is; |
| 1042 | float mse = 0; |
| 1043 | for (int i = 0; i < n; ++i) { |
| 1044 | int l = nearest_int(iscale_is*x[i]); |
| 1045 | l = MIN(nmax, l)((nmax) < (l) ? (nmax) : (l)); |
| 1046 | float diff = x[i] - scale_is*l; |
| 1047 | float w = quant_weights[i]; |
| 1048 | mse += w*diff*diff; |
| 1049 | } |
| 1050 | if (mse < best_mse) { |
| 1051 | best_mse = mse; |
| 1052 | iscale = iscale_is; |
| 1053 | } |
| 1054 | } |
| 1055 | float sumlx = 0; |
| 1056 | float suml2 = 0; |
| 1057 | for (int i = 0; i < n; ++i) { |
| 1058 | int l = nearest_int(iscale * x[i]); |
| 1059 | l = MIN(nmax, l)((nmax) < (l) ? (nmax) : (l)); |
| 1060 | L[i] = l; |
| 1061 | float w = quant_weights[i]; |
| 1062 | sumlx += w*x[i]*l; |
| 1063 | suml2 += w*l*l; |
| 1064 | } |
| 1065 | for (int itry = 0; itry < 5; ++itry) { |
| 1066 | int n_changed = 0; |
| 1067 | for (int i = 0; i < n; ++i) { |
| 1068 | float w = quant_weights[i]; |
| 1069 | float slx = sumlx - w*x[i]*L[i]; |
| 1070 | float sl2 = suml2 - w*L[i]*L[i]; |
| 1071 | if (slx > 0 && sl2 > 0) { |
| 1072 | int new_l = nearest_int(x[i] * sl2 / slx); |
| 1073 | new_l = MIN(nmax, new_l)((nmax) < (new_l) ? (nmax) : (new_l)); |
| 1074 | if (new_l != L[i]) { |
| 1075 | slx += w*x[i]*new_l; |
| 1076 | sl2 += w*new_l*new_l; |
| 1077 | if (slx*slx*suml2 > sumlx*sumlx*sl2) { |
| 1078 | L[i] = new_l; sumlx = slx; suml2 = sl2; |
| 1079 | ++n_changed; |
| 1080 | } |
| 1081 | } |
| 1082 | } |
| 1083 | } |
| 1084 | if (!n_changed) { |
| 1085 | break; |
| 1086 | } |
| 1087 | } |
| 1088 | return suml2 > 0.0f ? sumlx / suml2 : 0.0f; |
| 1089 | } |
| 1090 | |
| 1091 | static void quantize_row_q2_K_impl(const float * GGML_RESTRICTrestrict x, block_q2_K * GGML_RESTRICTrestrict y, int k, const float * GGML_RESTRICTrestrict quant_weights) { |
| 1092 | GGML_ASSERT(quant_weights)if (!(quant_weights)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1092, "GGML_ASSERT(%s) failed", "quant_weights"); |
| 1093 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1093, __extension__ __PRETTY_FUNCTION__); })); |
| 1094 | const int nb = k / QK_K256; |
| 1095 | const bool_Bool requantize = true1; |
| 1096 | |
| 1097 | uint8_t L[QK_K256]; |
| 1098 | uint8_t Laux[16]; |
| 1099 | float mins[QK_K256/16]; |
| 1100 | float scales[QK_K256/16]; |
| 1101 | float sw[QK_K256/16]; |
| 1102 | float weight[16]; |
| 1103 | uint8_t Ls[QK_K256/16], Lm[QK_K256/16]; |
| 1104 | |
| 1105 | for (int i = 0; i < nb; i++) { |
| 1106 | memset(sw, 0, QK_K256/16*sizeof(float)); |
| 1107 | float sumx2 = 0; |
| 1108 | for (int j = 0; j < QK_K256; ++j) sumx2 += x[j]*x[j]; |
| 1109 | float sigma2 = sumx2/QK_K256; |
| 1110 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1111 | const float * GGML_RESTRICTrestrict qw = quant_weights + QK_K256 * i + 16*j; |
| 1112 | for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j + l]*x[16*j + l]); |
| 1113 | for (int l = 0; l < QK_K256/16; ++l) sw[j] += weight[l]; |
| 1114 | scales[j] = make_qkx3_quants(16, 3, x + 16*j, weight, L + 16*j, &mins[j], Laux, -0.9f, 0.05f, 36, false0); |
| 1115 | } |
| 1116 | |
| 1117 | float dm, mm; |
| 1118 | dm = make_qp_quants(QK_K256/16, 15, scales, Ls, sw); |
| 1119 | mm = make_qp_quants(QK_K256/16, 15, mins, Lm, sw); |
| 1120 | |
| 1121 | y[i].d = GGML_FP32_TO_FP16(dm)ggml_compute_fp32_to_fp16(dm); |
| 1122 | y[i].dmin = GGML_FP32_TO_FP16(mm)ggml_compute_fp32_to_fp16(mm); |
| 1123 | dm = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d); |
| 1124 | mm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin); |
| 1125 | |
| 1126 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1127 | y[i].scales[j] = Ls[j] | (Lm[j] << 4); |
| 1128 | } |
| 1129 | |
| 1130 | if (requantize) { |
| 1131 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1132 | const float d = dm * (y[i].scales[j] & 0xF); |
| 1133 | if (!d) continue; |
| 1134 | const float m = mm * (y[i].scales[j] >> 4); |
| 1135 | for (int ii = 0; ii < 16; ++ii) { |
| 1136 | int l = nearest_int((x[16*j + ii] + m)/d); |
| 1137 | l = MAX(0, MIN(3, l))((0) > (((3) < (l) ? (3) : (l))) ? (0) : (((3) < (l) ? (3) : (l)))); |
| 1138 | L[16*j + ii] = l; |
| 1139 | } |
| 1140 | } |
| 1141 | } |
| 1142 | |
| 1143 | for (int j = 0; j < QK_K256; j += 128) { |
| 1144 | for (int l = 0; l < 32; ++l) { |
| 1145 | y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); |
| 1146 | } |
| 1147 | } |
| 1148 | |
| 1149 | x += QK_K256; |
| 1150 | } |
| 1151 | } |
| 1152 | |
| 1153 | size_t quantize_q2_K(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 1154 | size_t row_size = ggml_row_size(GGML_TYPE_Q2_K, n_per_row); |
| 1155 | if (!quant_weights) { |
| 1156 | quantize_row_q2_K_ref(src, dst, (int64_t)nrow*n_per_row); |
| 1157 | } |
| 1158 | else { |
| 1159 | char * qrow = (char *)dst; |
| 1160 | for (int64_t row = 0; row < nrow; ++row) { |
| 1161 | quantize_row_q2_K_impl(src, (block_q2_K*)qrow, n_per_row, quant_weights); |
| 1162 | src += n_per_row; |
| 1163 | qrow += row_size; |
| 1164 | } |
| 1165 | } |
| 1166 | return nrow * row_size; |
| 1167 | } |
| 1168 | |
| 1169 | //========================= 3-bit (de)-quantization |
| 1170 | |
| 1171 | void quantize_row_q3_K_ref(const float * GGML_RESTRICTrestrict x, block_q3_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 1172 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1172, __extension__ __PRETTY_FUNCTION__); })); |
| 1173 | const int nb = k / QK_K256; |
| 1174 | |
| 1175 | int8_t L[QK_K256]; |
| 1176 | float scales[QK_K256 / 16]; |
| 1177 | |
| 1178 | for (int i = 0; i < nb; i++) { |
| 1179 | |
| 1180 | float max_scale = 0; |
| 1181 | float amax = 0; |
| 1182 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1183 | scales[j] = make_q3_quants(16, 4, x + 16*j, L + 16*j, true1); |
| 1184 | float scale = fabsf(scales[j]); |
| 1185 | if (scale > amax) { |
| 1186 | amax = scale; max_scale = scales[j]; |
| 1187 | } |
| 1188 | } |
| 1189 | |
| 1190 | memset(y[i].scales, 0, 12); |
| 1191 | if (max_scale) { |
| 1192 | float iscale = -32.f/max_scale; |
| 1193 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1194 | int8_t l = nearest_int(iscale*scales[j]); |
| 1195 | l = MAX(-32, MIN(31, l))((-32) > (((31) < (l) ? (31) : (l))) ? (-32) : (((31) < (l) ? (31) : (l)))) + 32; |
| 1196 | if (j < 8) { |
| 1197 | y[i].scales[j] = l & 0xF; |
| 1198 | } else { |
| 1199 | y[i].scales[j-8] |= ((l & 0xF) << 4); |
| 1200 | } |
| 1201 | l >>= 4; |
| 1202 | y[i].scales[j%4 + 8] |= (l << (2*(j/4))); |
| 1203 | } |
| 1204 | y[i].d = GGML_FP32_TO_FP16(1/iscale)ggml_compute_fp32_to_fp16(1/iscale); |
| 1205 | } else { |
| 1206 | y[i].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 1207 | } |
| 1208 | |
| 1209 | int8_t sc; |
| 1210 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1211 | sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4; |
| 1212 | sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32; |
| 1213 | float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1214 | if (!d) { |
| 1215 | continue; |
| 1216 | } |
| 1217 | for (int ii = 0; ii < 16; ++ii) { |
| 1218 | int l = nearest_int(x[16*j + ii]/d); |
| 1219 | l = MAX(-4, MIN(3, l))((-4) > (((3) < (l) ? (3) : (l))) ? (-4) : (((3) < ( l) ? (3) : (l)))); |
| 1220 | L[16*j + ii] = l + 4; |
| 1221 | } |
| 1222 | } |
| 1223 | |
| 1224 | memset(y[i].hmask, 0, QK_K256/8); |
| 1225 | // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc. |
| 1226 | int m = 0; |
| 1227 | uint8_t hm = 1; |
| 1228 | for (int j = 0; j < QK_K256; ++j) { |
| 1229 | if (L[j] > 3) { |
| 1230 | y[i].hmask[m] |= hm; |
| 1231 | L[j] -= 4; |
| 1232 | } |
| 1233 | if (++m == QK_K256/8) { |
| 1234 | m = 0; hm <<= 1; |
| 1235 | } |
| 1236 | } |
| 1237 | for (int j = 0; j < QK_K256; j += 128) { |
| 1238 | for (int l = 0; l < 32; ++l) { |
| 1239 | y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); |
| 1240 | } |
| 1241 | } |
| 1242 | |
| 1243 | x += QK_K256; |
| 1244 | } |
| 1245 | } |
| 1246 | |
| 1247 | void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 1248 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1248, __extension__ __PRETTY_FUNCTION__); })); |
| 1249 | const int nb = k / QK_K256; |
| 1250 | |
| 1251 | const uint32_t kmask1 = 0x03030303; |
| 1252 | const uint32_t kmask2 = 0x0f0f0f0f; |
| 1253 | |
| 1254 | uint32_t aux[4]; |
| 1255 | const int8_t * scales = (const int8_t*)aux; |
| 1256 | |
| 1257 | for (int i = 0; i < nb; i++) { |
| 1258 | |
| 1259 | const float d_all = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 1260 | |
| 1261 | const uint8_t * GGML_RESTRICTrestrict q = x[i].qs; |
| 1262 | const uint8_t * GGML_RESTRICTrestrict hm = x[i].hmask; |
| 1263 | uint8_t m = 1; |
| 1264 | |
| 1265 | memcpy(aux, x[i].scales, 12); |
| 1266 | uint32_t tmp = aux[2]; |
| 1267 | aux[2] = ((aux[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); |
| 1268 | aux[3] = ((aux[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); |
| 1269 | aux[0] = (aux[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); |
| 1270 | aux[1] = (aux[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); |
| 1271 | |
| 1272 | int is = 0; |
| 1273 | float dl; |
| 1274 | for (int n = 0; n < QK_K256; n += 128) { |
| 1275 | int shift = 0; |
| 1276 | for (int j = 0; j < 4; ++j) { |
| 1277 | |
| 1278 | dl = d_all * (scales[is++] - 32); |
| 1279 | for (int l = 0; l < 16; ++l) { |
| 1280 | *y++ = dl * ((int8_t)((q[l+ 0] >> shift) & 3) - ((hm[l+ 0] & m) ? 0 : 4)); |
| 1281 | } |
| 1282 | |
| 1283 | dl = d_all * (scales[is++] - 32); |
| 1284 | for (int l = 0; l < 16; ++l) { |
| 1285 | *y++ = dl * ((int8_t)((q[l+16] >> shift) & 3) - ((hm[l+16] & m) ? 0 : 4)); |
| 1286 | } |
| 1287 | |
| 1288 | shift += 2; |
| 1289 | m <<= 1; |
| 1290 | } |
| 1291 | q += 32; |
| 1292 | } |
| 1293 | |
| 1294 | } |
| 1295 | } |
| 1296 | |
| 1297 | static void quantize_row_q3_K_impl(const float * GGML_RESTRICTrestrict x, block_q3_K * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * GGML_RESTRICTrestrict quant_weights) { |
| 1298 | assert(n_per_row % QK_K == 0)((void) sizeof (__assert_single_arg (n_per_row % 256 == 0)), __extension__ ({ if (n_per_row % 256 == 0) ; else __assert_fail ("n_per_row % QK_K == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1298, __extension__ __PRETTY_FUNCTION__); })); |
| 1299 | const int nb = n_per_row / QK_K256; |
| 1300 | |
| 1301 | int8_t L[QK_K256]; |
| 1302 | float scales[QK_K256 / 16]; |
| 1303 | float weight[16]; |
| 1304 | float sw[QK_K256 / 16]; |
| 1305 | int8_t Ls[QK_K256 / 16]; |
| 1306 | |
| 1307 | for (int i = 0; i < nb; i++) { |
| 1308 | |
| 1309 | float sumx2 = 0; |
| 1310 | for (int j = 0; j < QK_K256; ++j) sumx2 += x[j]*x[j]; |
| 1311 | float sigma2 = 2*sumx2/QK_K256; |
| 1312 | |
| 1313 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1314 | if (quant_weights) { |
| 1315 | const float * qw = quant_weights + QK_K256 * i + 16*j; |
| 1316 | for (int l = 0; l < 16; ++l) weight[l] = qw[l] * sqrtf(sigma2 + x[16*j+l]*x[16*j+l]); |
| 1317 | } else { |
| 1318 | for (int l = 0; l < 16; ++l) weight[l] = x[16*j+l]*x[16*j+l]; |
| 1319 | } |
| 1320 | float sumw = 0; |
| 1321 | for (int l = 0; l < 16; ++l) sumw += weight[l]; |
| 1322 | sw[j] = sumw; |
| 1323 | |
| 1324 | scales[j] = make_qx_quants(16, 4, x + 16*j, L + 16*j, 1, weight); |
| 1325 | |
| 1326 | } |
| 1327 | |
| 1328 | memset(y[i].scales, 0, 12); |
| 1329 | |
| 1330 | float d_block = make_qx_quants(QK_K256/16, 32, scales, Ls, 1, sw); |
| 1331 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1332 | int l = Ls[j]; |
| 1333 | if (j < 8) { |
| 1334 | y[i].scales[j] = l & 0xF; |
| 1335 | } else { |
| 1336 | y[i].scales[j-8] |= ((l & 0xF) << 4); |
| 1337 | } |
| 1338 | l >>= 4; |
| 1339 | y[i].scales[j%4 + 8] |= (l << (2*(j/4))); |
| 1340 | } |
| 1341 | y[i].d = GGML_FP32_TO_FP16(d_block)ggml_compute_fp32_to_fp16(d_block); |
| 1342 | |
| 1343 | int8_t sc; |
| 1344 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1345 | sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j-8] >> 4; |
| 1346 | sc = (sc | (((y[i].scales[8 + j%4] >> (2*(j/4))) & 3) << 4)) - 32; |
| 1347 | float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1348 | if (!d) { |
| 1349 | continue; |
| 1350 | } |
| 1351 | for (int ii = 0; ii < 16; ++ii) { |
| 1352 | int l = nearest_int(x[16*j + ii]/d); |
| 1353 | l = MAX(-4, MIN(3, l))((-4) > (((3) < (l) ? (3) : (l))) ? (-4) : (((3) < ( l) ? (3) : (l)))); |
| 1354 | L[16*j + ii] = l + 4; |
| 1355 | } |
| 1356 | } |
| 1357 | |
| 1358 | memset(y[i].hmask, 0, QK_K256/8); |
| 1359 | // We put the high-bit for the 1st 8 quants into bit 0, the next 8 into bit 1, etc. |
| 1360 | int m = 0; |
| 1361 | uint8_t hm = 1; |
| 1362 | for (int j = 0; j < QK_K256; ++j) { |
| 1363 | if (L[j] > 3) { |
| 1364 | y[i].hmask[m] |= hm; |
| 1365 | L[j] -= 4; |
| 1366 | } |
| 1367 | if (++m == QK_K256/8) { |
| 1368 | m = 0; hm <<= 1; |
| 1369 | } |
| 1370 | } |
| 1371 | for (int j = 0; j < QK_K256; j += 128) { |
| 1372 | for (int l = 0; l < 32; ++l) { |
| 1373 | y[i].qs[j/4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); |
| 1374 | } |
| 1375 | } |
| 1376 | |
| 1377 | x += QK_K256; |
| 1378 | } |
| 1379 | } |
| 1380 | |
| 1381 | size_t quantize_q3_K(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 1382 | size_t row_size = ggml_row_size(GGML_TYPE_Q3_K, n_per_row); |
| 1383 | if (!quant_weights) { |
| 1384 | quantize_row_q3_K_ref(src, dst, (int64_t)nrow*n_per_row); |
| 1385 | } |
| 1386 | else { |
| 1387 | char * qrow = (char *)dst; |
| 1388 | for (int64_t row = 0; row < nrow; ++row) { |
| 1389 | quantize_row_q3_K_impl(src, (block_q3_K*)qrow, n_per_row, quant_weights); |
| 1390 | src += n_per_row; |
| 1391 | qrow += row_size; |
| 1392 | } |
| 1393 | } |
| 1394 | return nrow * row_size; |
| 1395 | } |
| 1396 | |
| 1397 | // ====================== 4-bit (de)-quantization |
| 1398 | |
| 1399 | void quantize_row_q4_K_ref(const float * GGML_RESTRICTrestrict x, block_q4_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 1400 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1400, __extension__ __PRETTY_FUNCTION__); })); |
| 1401 | const int nb = k / QK_K256; |
| 1402 | |
| 1403 | uint8_t L[QK_K256]; |
| 1404 | uint8_t Laux[32]; |
| 1405 | float weights[32]; |
| 1406 | float mins[QK_K256/32]; |
| 1407 | float scales[QK_K256/32]; |
| 1408 | |
| 1409 | for (int i = 0; i < nb; i++) { |
| 1410 | float max_scale = 0; // as we are deducting the min, scales are always positive |
| 1411 | float max_min = 0; |
| 1412 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1413 | //scales[j] = make_qkx1_quants(32, 15, x + 32*j, L + 32*j, &mins[j], 9, 0.5f); |
| 1414 | float sum_x2 = 0; |
| 1415 | for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l]; |
| 1416 | float av_x = sqrtf(sum_x2/32); |
| 1417 | for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); |
| 1418 | scales[j] = make_qkx2_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -1.f, 0.1f, 20, false0); |
| 1419 | float scale = scales[j]; |
| 1420 | if (scale > max_scale) { |
| 1421 | max_scale = scale; |
| 1422 | } |
| 1423 | float min = mins[j]; |
| 1424 | if (min > max_min) { |
| 1425 | max_min = min; |
| 1426 | } |
| 1427 | } |
| 1428 | |
| 1429 | float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; |
| 1430 | float inv_min = max_min > 0 ? 63.f/max_min : 0.f; |
| 1431 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1432 | uint8_t ls = nearest_int(inv_scale*scales[j]); |
| 1433 | uint8_t lm = nearest_int(inv_min*mins[j]); |
| 1434 | ls = MIN(63, ls)((63) < (ls) ? (63) : (ls)); |
| 1435 | lm = MIN(63, lm)((63) < (lm) ? (63) : (lm)); |
| 1436 | if (j < 4) { |
| 1437 | y[i].scales[j] = ls; |
| 1438 | y[i].scales[j+4] = lm; |
| 1439 | } else { |
| 1440 | y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); |
| 1441 | y[i].scales[j-4] |= ((ls >> 4) << 6); |
| 1442 | y[i].scales[j-0] |= ((lm >> 4) << 6); |
| 1443 | } |
| 1444 | } |
| 1445 | y[i].d = GGML_FP32_TO_FP16(max_scale/63.f)ggml_compute_fp32_to_fp16(max_scale/63.f); |
| 1446 | y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f)ggml_compute_fp32_to_fp16(max_min/63.f); |
| 1447 | |
| 1448 | uint8_t sc, m; |
| 1449 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1450 | get_scale_min_k4(j, y[i].scales, &sc, &m); |
| 1451 | const float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1452 | if (!d) continue; |
| 1453 | const float dm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin) * m; |
| 1454 | for (int ii = 0; ii < 32; ++ii) { |
| 1455 | int l = nearest_int((x[32*j + ii] + dm)/d); |
| 1456 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 1457 | L[32*j + ii] = l; |
| 1458 | } |
| 1459 | } |
| 1460 | |
| 1461 | uint8_t * q = y[i].qs; |
| 1462 | for (int j = 0; j < QK_K256; j += 64) { |
| 1463 | for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4); |
| 1464 | q += 32; |
| 1465 | } |
| 1466 | |
| 1467 | x += QK_K256; |
| 1468 | } |
| 1469 | } |
| 1470 | |
| 1471 | void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 1472 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1472, __extension__ __PRETTY_FUNCTION__); })); |
| 1473 | const int nb = k / QK_K256; |
| 1474 | |
| 1475 | for (int i = 0; i < nb; i++) { |
| 1476 | const uint8_t * q = x[i].qs; |
| 1477 | |
| 1478 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 1479 | const float min = GGML_FP16_TO_FP32(x[i].dmin)ggml_compute_fp16_to_fp32(x[i].dmin); |
| 1480 | |
| 1481 | int is = 0; |
| 1482 | uint8_t sc, m; |
| 1483 | for (int j = 0; j < QK_K256; j += 64) { |
| 1484 | get_scale_min_k4(is + 0, x[i].scales, &sc, &m); |
| 1485 | const float d1 = d * sc; const float m1 = min * m; |
| 1486 | get_scale_min_k4(is + 1, x[i].scales, &sc, &m); |
| 1487 | const float d2 = d * sc; const float m2 = min * m; |
| 1488 | for (int l = 0; l < 32; ++l) *y++ = d1 * (q[l] & 0xF) - m1; |
| 1489 | for (int l = 0; l < 32; ++l) *y++ = d2 * (q[l] >> 4) - m2; |
| 1490 | q += 32; is += 2; |
| 1491 | } |
| 1492 | } |
| 1493 | } |
| 1494 | |
| 1495 | static void quantize_row_q4_K_impl(const float * GGML_RESTRICTrestrict x, block_q4_K * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 1496 | assert(n_per_row % QK_K == 0)((void) sizeof (__assert_single_arg (n_per_row % 256 == 0)), __extension__ ({ if (n_per_row % 256 == 0) ; else __assert_fail ("n_per_row % QK_K == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1496, __extension__ __PRETTY_FUNCTION__); })); |
| 1497 | const int64_t nb = n_per_row / QK_K256; |
| 1498 | |
| 1499 | uint8_t L[QK_K256]; |
| 1500 | uint8_t Laux[32]; |
| 1501 | uint8_t Ls[QK_K256/32]; |
| 1502 | uint8_t Lm[QK_K256/32]; |
| 1503 | float weights[32]; |
| 1504 | float sw[QK_K256/32]; |
| 1505 | float mins[QK_K256/32]; |
| 1506 | float scales[QK_K256/32]; |
| 1507 | |
| 1508 | for (int i = 0; i < nb; i++) { |
| 1509 | |
| 1510 | float sum_x2 = 0; |
| 1511 | for (int l = 0; l < QK_K256; ++l) sum_x2 += x[l] * x[l]; |
| 1512 | float sigma2 = 2*sum_x2/QK_K256; |
| 1513 | float av_x = sqrtf(sigma2); |
| 1514 | |
| 1515 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1516 | if (quant_weights) { |
| 1517 | const float * qw = quant_weights + QK_K256*i + 32*j; |
| 1518 | for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]); |
| 1519 | } else { |
| 1520 | for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); |
| 1521 | } |
| 1522 | float sumw = 0; |
| 1523 | for (int l = 0; l < 32; ++l) sumw += weights[l]; |
| 1524 | sw[j] = sumw; |
| 1525 | scales[j] = make_qkx3_quants(32, 15, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false0); |
| 1526 | } |
| 1527 | |
| 1528 | float d_block = make_qp_quants(QK_K256/32, 63, scales, Ls, sw); |
| 1529 | float m_block = make_qp_quants(QK_K256/32, 63, mins, Lm, sw); |
| 1530 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1531 | uint8_t ls = Ls[j]; |
| 1532 | uint8_t lm = Lm[j]; |
| 1533 | if (j < 4) { |
| 1534 | y[i].scales[j] = ls; |
| 1535 | y[i].scales[j+4] = lm; |
| 1536 | } else { |
| 1537 | y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); |
| 1538 | y[i].scales[j-4] |= ((ls >> 4) << 6); |
| 1539 | y[i].scales[j-0] |= ((lm >> 4) << 6); |
| 1540 | } |
| 1541 | } |
| 1542 | y[i].d = GGML_FP32_TO_FP16(d_block)ggml_compute_fp32_to_fp16(d_block); |
| 1543 | y[i].dmin = GGML_FP32_TO_FP16(m_block)ggml_compute_fp32_to_fp16(m_block); |
| 1544 | |
| 1545 | uint8_t sc, m; |
| 1546 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1547 | get_scale_min_k4(j, y[i].scales, &sc, &m); |
| 1548 | const float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1549 | if (!d) continue; |
| 1550 | const float dm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin) * m; |
| 1551 | for (int ii = 0; ii < 32; ++ii) { |
| 1552 | int l = nearest_int((x[32*j + ii] + dm)/d); |
| 1553 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 1554 | L[32*j + ii] = l; |
| 1555 | } |
| 1556 | } |
| 1557 | uint8_t * q = y[i].qs; |
| 1558 | for (int j = 0; j < QK_K256; j += 64) { |
| 1559 | for (int l = 0; l < 32; ++l) q[l] = L[j + l] | (L[j + l + 32] << 4); |
| 1560 | q += 32; |
| 1561 | } |
| 1562 | |
| 1563 | x += QK_K256; |
| 1564 | |
| 1565 | } |
| 1566 | } |
| 1567 | |
| 1568 | size_t quantize_q4_K(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 1569 | size_t row_size = ggml_row_size(GGML_TYPE_Q4_K, n_per_row); |
| 1570 | if (!quant_weights) { |
| 1571 | quantize_row_q4_K_ref(src, dst, (int64_t)nrow*n_per_row); |
| 1572 | } |
| 1573 | else { |
| 1574 | char * qrow = (char *)dst; |
| 1575 | for (int64_t row = 0; row < nrow; ++row) { |
| 1576 | quantize_row_q4_K_impl(src, (block_q4_K*)qrow, n_per_row, quant_weights); |
| 1577 | src += n_per_row; |
| 1578 | qrow += row_size; |
| 1579 | } |
| 1580 | } |
| 1581 | return nrow * row_size; |
| 1582 | } |
| 1583 | |
| 1584 | // ====================== 5-bit (de)-quantization |
| 1585 | |
| 1586 | void quantize_row_q5_K_ref(const float * GGML_RESTRICTrestrict x, block_q5_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 1587 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1587, __extension__ __PRETTY_FUNCTION__); })); |
| 1588 | const int64_t nb = k / QK_K256; |
| 1589 | |
| 1590 | uint8_t L[QK_K256]; |
| 1591 | float mins[QK_K256/32]; |
| 1592 | float scales[QK_K256/32]; |
| 1593 | float weights[32]; |
| 1594 | uint8_t Laux[32]; |
| 1595 | |
| 1596 | for (int i = 0; i < nb; i++) { |
| 1597 | float max_scale = 0; // as we are deducting the min, scales are always positive |
| 1598 | float max_min = 0; |
| 1599 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1600 | //scales[j] = make_qkx1_quants(32, 31, x + 32*j, L + 32*j, &mins[j], 9, 0.5f); |
| 1601 | float sum_x2 = 0; |
| 1602 | for (int l = 0; l < 32; ++l) sum_x2 += x[32*j + l] * x[32*j + l]; |
| 1603 | float av_x = sqrtf(sum_x2/32); |
| 1604 | for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); |
| 1605 | scales[j] = make_qkx2_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.5f, 0.1f, 15, false0); |
| 1606 | float scale = scales[j]; |
| 1607 | if (scale > max_scale) { |
| 1608 | max_scale = scale; |
| 1609 | } |
| 1610 | float min = mins[j]; |
| 1611 | if (min > max_min) { |
| 1612 | max_min = min; |
| 1613 | } |
| 1614 | } |
| 1615 | |
| 1616 | float inv_scale = max_scale > 0 ? 63.f/max_scale : 0.f; |
| 1617 | float inv_min = max_min > 0 ? 63.f/max_min : 0.f; |
| 1618 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1619 | uint8_t ls = nearest_int(inv_scale*scales[j]); |
| 1620 | uint8_t lm = nearest_int(inv_min*mins[j]); |
| 1621 | ls = MIN(63, ls)((63) < (ls) ? (63) : (ls)); |
| 1622 | lm = MIN(63, lm)((63) < (lm) ? (63) : (lm)); |
| 1623 | if (j < 4) { |
| 1624 | y[i].scales[j] = ls; |
| 1625 | y[i].scales[j+4] = lm; |
| 1626 | } else { |
| 1627 | y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); |
| 1628 | y[i].scales[j-4] |= ((ls >> 4) << 6); |
| 1629 | y[i].scales[j-0] |= ((lm >> 4) << 6); |
| 1630 | } |
| 1631 | } |
| 1632 | y[i].d = GGML_FP32_TO_FP16(max_scale/63.f)ggml_compute_fp32_to_fp16(max_scale/63.f); |
| 1633 | y[i].dmin = GGML_FP32_TO_FP16(max_min/63.f)ggml_compute_fp32_to_fp16(max_min/63.f); |
| 1634 | |
| 1635 | uint8_t sc, m; |
| 1636 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1637 | get_scale_min_k4(j, y[i].scales, &sc, &m); |
| 1638 | const float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1639 | if (!d) continue; |
| 1640 | const float dm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin) * m; |
| 1641 | for (int ii = 0; ii < 32; ++ii) { |
| 1642 | int l = nearest_int((x[32*j + ii] + dm)/d); |
| 1643 | l = MAX(0, MIN(31, l))((0) > (((31) < (l) ? (31) : (l))) ? (0) : (((31) < ( l) ? (31) : (l)))); |
| 1644 | L[32*j + ii] = l; |
| 1645 | } |
| 1646 | } |
| 1647 | |
| 1648 | uint8_t * GGML_RESTRICTrestrict qh = y[i].qh; |
| 1649 | uint8_t * GGML_RESTRICTrestrict ql = y[i].qs; |
| 1650 | memset(qh, 0, QK_K256/8); |
| 1651 | |
| 1652 | uint8_t m1 = 1, m2 = 2; |
| 1653 | for (int n = 0; n < QK_K256; n += 64) { |
| 1654 | for (int j = 0; j < 32; ++j) { |
| 1655 | int l1 = L[n + j]; |
| 1656 | if (l1 > 15) { |
| 1657 | l1 -= 16; qh[j] |= m1; |
| 1658 | } |
| 1659 | int l2 = L[n + j + 32]; |
| 1660 | if (l2 > 15) { |
| 1661 | l2 -= 16; qh[j] |= m2; |
| 1662 | } |
| 1663 | ql[j] = l1 | (l2 << 4); |
| 1664 | } |
| 1665 | m1 <<= 2; m2 <<= 2; |
| 1666 | ql += 32; |
| 1667 | } |
| 1668 | |
| 1669 | x += QK_K256; |
| 1670 | } |
| 1671 | } |
| 1672 | |
| 1673 | void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 1674 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1674, __extension__ __PRETTY_FUNCTION__); })); |
| 1675 | const int64_t nb = k / QK_K256; |
| 1676 | |
| 1677 | for (int i = 0; i < nb; i++) { |
| 1678 | const uint8_t * ql = x[i].qs; |
| 1679 | const uint8_t * qh = x[i].qh; |
| 1680 | |
| 1681 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 1682 | const float min = GGML_FP16_TO_FP32(x[i].dmin)ggml_compute_fp16_to_fp32(x[i].dmin); |
| 1683 | |
| 1684 | int is = 0; |
| 1685 | uint8_t sc, m; |
| 1686 | uint8_t u1 = 1, u2 = 2; |
| 1687 | for (int j = 0; j < QK_K256; j += 64) { |
| 1688 | get_scale_min_k4(is + 0, x[i].scales, &sc, &m); |
| 1689 | const float d1 = d * sc; const float m1 = min * m; |
| 1690 | get_scale_min_k4(is + 1, x[i].scales, &sc, &m); |
| 1691 | const float d2 = d * sc; const float m2 = min * m; |
| 1692 | for (int l = 0; l < 32; ++l) *y++ = d1 * ((ql[l] & 0xF) + (qh[l] & u1 ? 16 : 0)) - m1; |
| 1693 | for (int l = 0; l < 32; ++l) *y++ = d2 * ((ql[l] >> 4) + (qh[l] & u2 ? 16 : 0)) - m2; |
| 1694 | ql += 32; is += 2; |
| 1695 | u1 <<= 2; u2 <<= 2; |
| 1696 | } |
| 1697 | } |
| 1698 | } |
| 1699 | |
| 1700 | static void quantize_row_q5_K_impl(const float * GGML_RESTRICTrestrict x, block_q5_K * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 1701 | assert(n_per_row % QK_K == 0)((void) sizeof (__assert_single_arg (n_per_row % 256 == 0)), __extension__ ({ if (n_per_row % 256 == 0) ; else __assert_fail ("n_per_row % QK_K == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1701, __extension__ __PRETTY_FUNCTION__); })); |
| 1702 | const int64_t nb = n_per_row / QK_K256; |
| 1703 | |
| 1704 | uint8_t L[QK_K256]; |
| 1705 | uint8_t Laux[32]; |
| 1706 | uint8_t Ls[QK_K256/32]; |
| 1707 | uint8_t Lm[QK_K256/32]; |
| 1708 | float mins[QK_K256/32]; |
| 1709 | float scales[QK_K256/32]; |
| 1710 | float sw[QK_K256/32]; |
| 1711 | float weights[32]; |
| 1712 | |
| 1713 | for (int i = 0; i < nb; i++) { |
| 1714 | |
| 1715 | float sum_x2 = 0; |
| 1716 | for (int l = 0; l < QK_K256; ++l) sum_x2 += x[l] * x[l]; |
| 1717 | float sigma2 = 2*sum_x2/QK_K256; |
| 1718 | float av_x = sqrtf(sigma2); |
| 1719 | |
| 1720 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1721 | if (quant_weights) { |
| 1722 | const float * qw = quant_weights + QK_K256*i + 32*j; |
| 1723 | for (int l = 0; l < 32; ++l) weights[l] = qw[l] * sqrtf(sigma2 + x[32*j + l]*x[32*j + l]); |
| 1724 | } else { |
| 1725 | for (int l = 0; l < 32; ++l) weights[l] = av_x + fabsf(x[32*j + l]); |
| 1726 | } |
| 1727 | float sumw = 0; |
| 1728 | for (int l = 0; l < 32; ++l) sumw += weights[l]; |
| 1729 | sw[j] = sumw; |
| 1730 | |
| 1731 | scales[j] = make_qkx3_quants(32, 31, x + 32*j, weights, L + 32*j, &mins[j], Laux, -0.9f, 0.05f, 36, false0); |
| 1732 | } |
| 1733 | |
| 1734 | float d_block = make_qp_quants(QK_K256/32, 63, scales, Ls, sw); |
| 1735 | float m_block = make_qp_quants(QK_K256/32, 63, mins, Lm, sw); |
| 1736 | |
| 1737 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1738 | uint8_t ls = Ls[j]; |
| 1739 | uint8_t lm = Lm[j]; |
| 1740 | ls = MIN(63, ls)((63) < (ls) ? (63) : (ls)); |
| 1741 | lm = MIN(63, lm)((63) < (lm) ? (63) : (lm)); |
| 1742 | if (j < 4) { |
| 1743 | y[i].scales[j] = ls; |
| 1744 | y[i].scales[j+4] = lm; |
| 1745 | } else { |
| 1746 | y[i].scales[j+4] = (ls & 0xF) | ((lm & 0xF) << 4); |
| 1747 | y[i].scales[j-4] |= ((ls >> 4) << 6); |
| 1748 | y[i].scales[j-0] |= ((lm >> 4) << 6); |
| 1749 | } |
| 1750 | } |
| 1751 | y[i].d = GGML_FP32_TO_FP16(d_block)ggml_compute_fp32_to_fp16(d_block); |
| 1752 | y[i].dmin = GGML_FP32_TO_FP16(m_block)ggml_compute_fp32_to_fp16(m_block); |
| 1753 | |
| 1754 | uint8_t sc, m; |
| 1755 | for (int j = 0; j < QK_K256/32; ++j) { |
| 1756 | get_scale_min_k4(j, y[i].scales, &sc, &m); |
| 1757 | const float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * sc; |
| 1758 | if (!d) continue; |
| 1759 | const float dm = GGML_FP16_TO_FP32(y[i].dmin)ggml_compute_fp16_to_fp32(y[i].dmin) * m; |
| 1760 | for (int ii = 0; ii < 32; ++ii) { |
| 1761 | int l = nearest_int((x[32*j + ii] + dm)/d); |
| 1762 | l = MAX(0, MIN(31, l))((0) > (((31) < (l) ? (31) : (l))) ? (0) : (((31) < ( l) ? (31) : (l)))); |
| 1763 | L[32*j + ii] = l; |
| 1764 | } |
| 1765 | } |
| 1766 | |
| 1767 | uint8_t * GGML_RESTRICTrestrict qh = y[i].qh; |
| 1768 | uint8_t * GGML_RESTRICTrestrict ql = y[i].qs; |
| 1769 | memset(qh, 0, QK_K256/8); |
| 1770 | |
| 1771 | uint8_t m1 = 1, m2 = 2; |
| 1772 | for (int n = 0; n < QK_K256; n += 64) { |
| 1773 | for (int j = 0; j < 32; ++j) { |
| 1774 | int l1 = L[n + j]; |
| 1775 | if (l1 > 15) { |
| 1776 | l1 -= 16; qh[j] |= m1; |
| 1777 | } |
| 1778 | int l2 = L[n + j + 32]; |
| 1779 | if (l2 > 15) { |
| 1780 | l2 -= 16; qh[j] |= m2; |
| 1781 | } |
| 1782 | ql[j] = l1 | (l2 << 4); |
| 1783 | } |
| 1784 | m1 <<= 2; m2 <<= 2; |
| 1785 | ql += 32; |
| 1786 | } |
| 1787 | |
| 1788 | x += QK_K256; |
| 1789 | |
| 1790 | } |
| 1791 | } |
| 1792 | |
| 1793 | size_t quantize_q5_K(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 1794 | size_t row_size = ggml_row_size(GGML_TYPE_Q5_K, n_per_row); |
| 1795 | if (!quant_weights) { |
| 1796 | quantize_row_q5_K_ref(src, dst, (int64_t)nrow*n_per_row); |
| 1797 | } |
| 1798 | else { |
| 1799 | char * qrow = (char *)dst; |
| 1800 | for (int64_t row = 0; row < nrow; ++row) { |
| 1801 | quantize_row_q5_K_impl(src, (block_q5_K*)qrow, n_per_row, quant_weights); |
| 1802 | src += n_per_row; |
| 1803 | qrow += row_size; |
| 1804 | } |
| 1805 | } |
| 1806 | return nrow * row_size; |
| 1807 | } |
| 1808 | |
| 1809 | // ====================== 6-bit (de)-quantization |
| 1810 | |
| 1811 | void quantize_row_q6_K_ref(const float * GGML_RESTRICTrestrict x, block_q6_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 1812 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1812, __extension__ __PRETTY_FUNCTION__); })); |
| 1813 | const int64_t nb = k / QK_K256; |
| 1814 | |
| 1815 | int8_t L[QK_K256]; |
| 1816 | float scales[QK_K256/16]; |
| 1817 | |
| 1818 | for (int i = 0; i < nb; i++) { |
| 1819 | |
| 1820 | float max_scale = 0; |
| 1821 | float max_abs_scale = 0; |
| 1822 | |
| 1823 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 1824 | |
| 1825 | const float scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL((void*)0)); |
| 1826 | scales[ib] = scale; |
| 1827 | |
| 1828 | const float abs_scale = fabsf(scale); |
| 1829 | if (abs_scale > max_abs_scale) { |
| 1830 | max_abs_scale = abs_scale; |
| 1831 | max_scale = scale; |
| 1832 | } |
| 1833 | |
| 1834 | } |
| 1835 | |
| 1836 | if (max_abs_scale < GROUP_MAX_EPS1e-15f) { |
| 1837 | memset(&y[i], 0, sizeof(block_q6_K)); |
| 1838 | y[i].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 1839 | x += QK_K256; |
| 1840 | continue; |
| 1841 | } |
| 1842 | |
| 1843 | float iscale = -128.f/max_scale; |
| 1844 | y[i].d = GGML_FP32_TO_FP16(1/iscale)ggml_compute_fp32_to_fp16(1/iscale); |
| 1845 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 1846 | y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]))((127) < (nearest_int(iscale*scales[ib])) ? (127) : (nearest_int (iscale*scales[ib]))); |
| 1847 | } |
| 1848 | |
| 1849 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1850 | float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * y[i].scales[j]; |
| 1851 | if (!d) { |
| 1852 | continue; |
| 1853 | } |
| 1854 | for (int ii = 0; ii < 16; ++ii) { |
| 1855 | int l = nearest_int(x[16*j + ii]/d); |
| 1856 | l = MAX(-32, MIN(31, l))((-32) > (((31) < (l) ? (31) : (l))) ? (-32) : (((31) < (l) ? (31) : (l)))); |
| 1857 | L[16*j + ii] = l + 32; |
| 1858 | } |
| 1859 | } |
| 1860 | |
| 1861 | uint8_t * GGML_RESTRICTrestrict ql = y[i].ql; |
| 1862 | uint8_t * GGML_RESTRICTrestrict qh = y[i].qh; |
| 1863 | for (int j = 0; j < QK_K256; j += 128) { |
| 1864 | for (int l = 0; l < 32; ++l) { |
| 1865 | const uint8_t q1 = L[j + l + 0] & 0xF; |
| 1866 | const uint8_t q2 = L[j + l + 32] & 0xF; |
| 1867 | const uint8_t q3 = L[j + l + 64] & 0xF; |
| 1868 | const uint8_t q4 = L[j + l + 96] & 0xF; |
| 1869 | ql[l+ 0] = q1 | (q3 << 4); |
| 1870 | ql[l+32] = q2 | (q4 << 4); |
| 1871 | qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6); |
| 1872 | } |
| 1873 | ql += 64; |
| 1874 | qh += 32; |
| 1875 | } |
| 1876 | |
| 1877 | x += QK_K256; |
| 1878 | } |
| 1879 | } |
| 1880 | |
| 1881 | void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 1882 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1882, __extension__ __PRETTY_FUNCTION__); })); |
| 1883 | const int64_t nb = k / QK_K256; |
| 1884 | |
| 1885 | for (int i = 0; i < nb; i++) { |
| 1886 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 1887 | |
| 1888 | const uint8_t * GGML_RESTRICTrestrict ql = x[i].ql; |
| 1889 | const uint8_t * GGML_RESTRICTrestrict qh = x[i].qh; |
| 1890 | const int8_t * GGML_RESTRICTrestrict sc = x[i].scales; |
| 1891 | |
| 1892 | for (int n = 0; n < QK_K256; n += 128) { |
| 1893 | for (int l = 0; l < 32; ++l) { |
| 1894 | int is = l/16; |
| 1895 | const int8_t q1 = (int8_t)((ql[l + 0] & 0xF) | (((qh[l] >> 0) & 3) << 4)) - 32; |
| 1896 | const int8_t q2 = (int8_t)((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) - 32; |
| 1897 | const int8_t q3 = (int8_t)((ql[l + 0] >> 4) | (((qh[l] >> 4) & 3) << 4)) - 32; |
| 1898 | const int8_t q4 = (int8_t)((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) - 32; |
| 1899 | y[l + 0] = d * sc[is + 0] * q1; |
| 1900 | y[l + 32] = d * sc[is + 2] * q2; |
| 1901 | y[l + 64] = d * sc[is + 4] * q3; |
| 1902 | y[l + 96] = d * sc[is + 6] * q4; |
| 1903 | } |
| 1904 | y += 128; |
| 1905 | ql += 64; |
| 1906 | qh += 32; |
| 1907 | sc += 8; |
| 1908 | } |
| 1909 | } |
| 1910 | } |
| 1911 | |
| 1912 | static void quantize_row_q6_K_impl(const float * GGML_RESTRICTrestrict x, block_q6_K * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 1913 | assert(n_per_row % QK_K == 0)((void) sizeof (__assert_single_arg (n_per_row % 256 == 0)), __extension__ ({ if (n_per_row % 256 == 0) ; else __assert_fail ("n_per_row % QK_K == 0" , "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 1913, __extension__ __PRETTY_FUNCTION__); })); |
| 1914 | const int64_t nb = n_per_row / QK_K256; |
| 1915 | |
| 1916 | int8_t L[QK_K256]; |
| 1917 | float scales[QK_K256/16]; |
| 1918 | //float weights[16]; |
| 1919 | |
| 1920 | for (int i = 0; i < nb; i++) { |
| 1921 | |
| 1922 | //float sum_x2 = 0; |
| 1923 | //for (int j = 0; j < QK_K; ++j) sum_x2 += x[j]*x[j]; |
| 1924 | //float sigma2 = sum_x2/QK_K; |
| 1925 | |
| 1926 | float max_scale = 0; |
| 1927 | float max_abs_scale = 0; |
| 1928 | |
| 1929 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 1930 | |
| 1931 | float scale; |
| 1932 | if (quant_weights) { |
| 1933 | const float * qw = quant_weights + QK_K256*i + 16*ib; |
| 1934 | //for (int j = 0; j < 16; ++j) weights[j] = qw[j] * sqrtf(sigma2 + x[16*ib + j]*x[16*ib + j]); |
| 1935 | //scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, weights); |
| 1936 | scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, qw); |
| 1937 | } else { |
| 1938 | scale = make_qx_quants(16, 32, x + 16*ib, L + 16*ib, 1, NULL((void*)0)); |
| 1939 | } |
| 1940 | scales[ib] = scale; |
| 1941 | |
| 1942 | const float abs_scale = fabsf(scale); |
| 1943 | if (abs_scale > max_abs_scale) { |
| 1944 | max_abs_scale = abs_scale; |
| 1945 | max_scale = scale; |
| 1946 | } |
| 1947 | |
| 1948 | } |
| 1949 | |
| 1950 | if (max_abs_scale < GROUP_MAX_EPS1e-15f) { |
| 1951 | memset(&y[i], 0, sizeof(block_q6_K)); |
| 1952 | y[i].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 1953 | x += QK_K256; |
| 1954 | continue; |
| 1955 | } |
| 1956 | |
| 1957 | float iscale = -128.f/max_scale; |
| 1958 | y[i].d = GGML_FP32_TO_FP16(1/iscale)ggml_compute_fp32_to_fp16(1/iscale); |
| 1959 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 1960 | y[i].scales[ib] = MIN(127, nearest_int(iscale*scales[ib]))((127) < (nearest_int(iscale*scales[ib])) ? (127) : (nearest_int (iscale*scales[ib]))); |
| 1961 | } |
| 1962 | |
| 1963 | for (int j = 0; j < QK_K256/16; ++j) { |
| 1964 | float d = GGML_FP16_TO_FP32(y[i].d)ggml_compute_fp16_to_fp32(y[i].d) * y[i].scales[j]; |
| 1965 | if (!d) { |
| 1966 | continue; |
| 1967 | } |
| 1968 | for (int ii = 0; ii < 16; ++ii) { |
| 1969 | int l = nearest_int(x[16*j + ii]/d); |
| 1970 | l = MAX(-32, MIN(31, l))((-32) > (((31) < (l) ? (31) : (l))) ? (-32) : (((31) < (l) ? (31) : (l)))); |
| 1971 | L[16*j + ii] = l + 32; |
| 1972 | } |
| 1973 | } |
| 1974 | |
| 1975 | uint8_t * GGML_RESTRICTrestrict ql = y[i].ql; |
| 1976 | uint8_t * GGML_RESTRICTrestrict qh = y[i].qh; |
| 1977 | for (int j = 0; j < QK_K256; j += 128) { |
| 1978 | for (int l = 0; l < 32; ++l) { |
| 1979 | const uint8_t q1 = L[j + l + 0] & 0xF; |
| 1980 | const uint8_t q2 = L[j + l + 32] & 0xF; |
| 1981 | const uint8_t q3 = L[j + l + 64] & 0xF; |
| 1982 | const uint8_t q4 = L[j + l + 96] & 0xF; |
| 1983 | ql[l+ 0] = q1 | (q3 << 4); |
| 1984 | ql[l+32] = q2 | (q4 << 4); |
| 1985 | qh[l] = (L[j + l] >> 4) | ((L[j + l + 32] >> 4) << 2) | ((L[j + l + 64] >> 4) << 4) | ((L[j + l + 96] >> 4) << 6); |
| 1986 | } |
| 1987 | ql += 64; |
| 1988 | qh += 32; |
| 1989 | } |
| 1990 | |
| 1991 | x += QK_K256; |
| 1992 | |
| 1993 | } |
| 1994 | } |
| 1995 | |
| 1996 | size_t quantize_q6_K(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 1997 | size_t row_size = ggml_row_size(GGML_TYPE_Q6_K, n_per_row); |
| 1998 | if (!quant_weights) { |
| 1999 | quantize_row_q6_K_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2000 | } |
| 2001 | else { |
| 2002 | char * qrow = (char *)dst; |
| 2003 | for (int64_t row = 0; row < nrow; ++row) { |
| 2004 | quantize_row_q6_K_impl(src, (block_q6_K*)qrow, n_per_row, quant_weights); |
| 2005 | src += n_per_row; |
| 2006 | qrow += row_size; |
| 2007 | } |
| 2008 | } |
| 2009 | return nrow * row_size; |
| 2010 | } |
| 2011 | |
| 2012 | static void quantize_row_q4_0_impl(const float * GGML_RESTRICTrestrict x, block_q4_0 * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 2013 | static_assert_Static_assert(QK4_032 == 32, "QK4_0 must be 32"); |
| 2014 | |
| 2015 | if (!quant_weights) { |
| 2016 | quantize_row_q4_0_ref(x, y, n_per_row); |
| 2017 | return; |
| 2018 | } |
| 2019 | |
| 2020 | float weight[QK4_032]; |
| 2021 | int8_t L[QK4_032]; |
| 2022 | |
| 2023 | float sum_x2 = 0; |
| 2024 | for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; |
| 2025 | float sigma2 = sum_x2/n_per_row; |
| 2026 | |
| 2027 | const int64_t nb = n_per_row/QK4_032; |
| 2028 | for (int ib = 0; ib < nb; ++ib) { |
| 2029 | const float * xb = x + QK4_032 * ib; |
| 2030 | const float * qw = quant_weights + QK4_032 * ib; |
| 2031 | for (int j = 0; j < QK4_032; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); |
| 2032 | float d = make_qx_quants(QK4_032, 8, xb, L, 1, weight); |
| 2033 | y[ib].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2034 | for (int j = 0; j < 16; ++j) { |
| 2035 | y[ib].qs[j] = L[j] | (L[j+16] << 4); |
| 2036 | } |
| 2037 | } |
| 2038 | } |
| 2039 | |
| 2040 | size_t quantize_q1_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2041 | if (!quant_weights) { |
| 2042 | quantize_row_q1_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2043 | return nrow * ggml_row_size(GGML_TYPE_Q1_0, n_per_row); |
| 2044 | } |
| 2045 | size_t row_size = ggml_row_size(GGML_TYPE_Q1_0, n_per_row); |
| 2046 | char * qrow = (char *)dst; |
| 2047 | for (int64_t row = 0; row < nrow; ++row) { |
| 2048 | quantize_row_q1_0_ref(src, (block_q1_0*)qrow, n_per_row); |
| 2049 | src += n_per_row; |
| 2050 | qrow += row_size; |
| 2051 | } |
| 2052 | return nrow * row_size; |
| 2053 | } |
| 2054 | |
| 2055 | |
| 2056 | size_t quantize_q4_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2057 | if (!quant_weights) { |
| 2058 | quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2059 | return nrow * ggml_row_size(GGML_TYPE_Q4_0, n_per_row); |
| 2060 | } |
| 2061 | size_t row_size = ggml_row_size(GGML_TYPE_Q4_0, n_per_row); |
| 2062 | char * qrow = (char *)dst; |
| 2063 | for (int64_t row = 0; row < nrow; ++row) { |
| 2064 | quantize_row_q4_0_impl(src, (block_q4_0*)qrow, n_per_row, quant_weights); |
| 2065 | src += n_per_row; |
| 2066 | qrow += row_size; |
| 2067 | } |
| 2068 | return nrow * row_size; |
| 2069 | } |
| 2070 | |
| 2071 | static void quantize_row_q4_1_impl(const float * GGML_RESTRICTrestrict x, block_q4_1 * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 2072 | static_assert_Static_assert(QK4_132 == 32, "QK4_1 must be 32"); |
| 2073 | |
| 2074 | if (!quant_weights) { |
| 2075 | quantize_row_q4_1_ref(x, y, n_per_row); |
| 2076 | return; |
| 2077 | } |
| 2078 | |
| 2079 | float weight[QK4_132]; |
| 2080 | uint8_t L[QK4_132], Laux[QK4_132]; |
| 2081 | |
| 2082 | float sum_x2 = 0; |
| 2083 | for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; |
| 2084 | float sigma2 = sum_x2/n_per_row; |
| 2085 | |
| 2086 | const int64_t nb = n_per_row/QK4_132; |
| 2087 | for (int ib = 0; ib < nb; ++ib) { |
| 2088 | const float * xb = x + QK4_132 * ib; |
| 2089 | const float * qw = quant_weights + QK4_132 * ib; |
| 2090 | for (int j = 0; j < QK4_132; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); |
| 2091 | float min; |
| 2092 | float d = make_qkx3_quants(QK4_132, 15, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false0); |
| 2093 | y[ib].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2094 | y[ib].m = GGML_FP32_TO_FP16(-min)ggml_compute_fp32_to_fp16(-min); |
| 2095 | for (int j = 0; j < 16; ++j) { |
| 2096 | y[ib].qs[j] = L[j] | (L[j+16] << 4); |
| 2097 | } |
| 2098 | } |
| 2099 | } |
| 2100 | |
| 2101 | size_t quantize_q4_1(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2102 | if (!quant_weights) { |
| 2103 | quantize_row_q4_1_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2104 | return nrow * ggml_row_size(GGML_TYPE_Q4_1, n_per_row); |
| 2105 | } |
| 2106 | size_t row_size = ggml_row_size(GGML_TYPE_Q4_1, n_per_row); |
| 2107 | char * qrow = (char *)dst; |
| 2108 | for (int64_t row = 0; row < nrow; ++row) { |
| 2109 | quantize_row_q4_1_impl(src, (block_q4_1*)qrow, n_per_row, quant_weights); |
| 2110 | src += n_per_row; |
| 2111 | qrow += row_size; |
| 2112 | } |
| 2113 | return nrow * row_size; |
| 2114 | } |
| 2115 | |
| 2116 | static void quantize_row_q5_0_impl(const float * GGML_RESTRICTrestrict x, block_q5_0 * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 2117 | static_assert_Static_assert(QK5_032 == 32, "QK5_0 must be 32"); |
| 2118 | |
| 2119 | if (!quant_weights) { |
| 2120 | quantize_row_q5_0_ref(x, y, n_per_row); |
| 2121 | return; |
| 2122 | } |
| 2123 | |
| 2124 | float weight[QK5_032]; |
| 2125 | int8_t L[QK5_032]; |
| 2126 | |
| 2127 | float sum_x2 = 0; |
| 2128 | for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; |
| 2129 | float sigma2 = sum_x2/n_per_row; |
| 2130 | |
| 2131 | const int64_t nb = n_per_row/QK5_032; |
| 2132 | for (int ib = 0; ib < nb; ++ib) { |
| 2133 | const float * xb = x + QK5_032 * ib; |
| 2134 | const float * qw = quant_weights + QK5_032 * ib; |
| 2135 | for (int j = 0; j < QK5_032; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); |
| 2136 | float d = make_qx_quants(QK5_032, 16, xb, L, 1, weight); |
| 2137 | y[ib].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2138 | |
| 2139 | uint32_t qh = 0; |
| 2140 | |
| 2141 | for (int j = 0; j < 16; ++j) { |
| 2142 | const uint8_t xi0 = L[j]; |
| 2143 | const uint8_t xi1 = L[j+16]; |
| 2144 | y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); |
| 2145 | |
| 2146 | // get the 5-th bit and store it in qh at the right position |
| 2147 | qh |= ((xi0 & 0x10u) >> 4) << (j + 0); |
| 2148 | qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_032/2); |
| 2149 | } |
| 2150 | |
| 2151 | memcpy(&y[ib].qh, &qh, sizeof(qh)); |
| 2152 | } |
| 2153 | } |
| 2154 | |
| 2155 | size_t quantize_q5_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2156 | if (!quant_weights) { |
| 2157 | quantize_row_q5_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2158 | return nrow * ggml_row_size(GGML_TYPE_Q5_0, n_per_row); |
| 2159 | } |
| 2160 | size_t row_size = ggml_row_size(GGML_TYPE_Q5_0, n_per_row); |
| 2161 | char * qrow = (char *)dst; |
| 2162 | for (int64_t row = 0; row < nrow; ++row) { |
| 2163 | quantize_row_q5_0_impl(src, (block_q5_0*)qrow, n_per_row, quant_weights); |
| 2164 | src += n_per_row; |
| 2165 | qrow += row_size; |
| 2166 | } |
| 2167 | return nrow * row_size; |
| 2168 | } |
| 2169 | |
| 2170 | static void quantize_row_q5_1_impl(const float * GGML_RESTRICTrestrict x, block_q5_1 * GGML_RESTRICTrestrict y, int64_t n_per_row, const float * quant_weights) { |
| 2171 | static_assert_Static_assert(QK5_132 == 32, "QK5_1 must be 32"); |
| 2172 | |
| 2173 | if (!quant_weights) { |
| 2174 | quantize_row_q5_1_ref(x, y, n_per_row); |
| 2175 | return; |
| 2176 | } |
| 2177 | |
| 2178 | float weight[QK5_132]; |
| 2179 | uint8_t L[QK5_132], Laux[QK5_132]; |
| 2180 | |
| 2181 | float sum_x2 = 0; |
| 2182 | for (int j = 0; j < n_per_row; ++j) sum_x2 += x[j]*x[j]; |
| 2183 | float sigma2 = sum_x2/n_per_row; |
| 2184 | |
| 2185 | const int64_t nb = n_per_row/QK5_132; |
| 2186 | for (int ib = 0; ib < nb; ++ib) { |
| 2187 | const float * xb = x + QK5_132 * ib; |
| 2188 | const float * qw = quant_weights + QK5_132 * ib; |
| 2189 | for (int j = 0; j < QK5_132; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); |
| 2190 | float min; |
| 2191 | float d = make_qkx3_quants(QK5_132, 31, xb, weight, L, &min, Laux, -0.9f, 0.05f, 36, false0); |
| 2192 | y[ib].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2193 | y[ib].m = GGML_FP32_TO_FP16(-min)ggml_compute_fp32_to_fp16(-min); |
| 2194 | |
| 2195 | uint32_t qh = 0; |
| 2196 | for (int j = 0; j < 16; ++j) { |
| 2197 | const uint8_t xi0 = L[j]; |
| 2198 | const uint8_t xi1 = L[j+16]; |
| 2199 | y[ib].qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); |
| 2200 | // get the 5-th bit and store it in qh at the right position |
| 2201 | qh |= ((xi0 & 0x10u) >> 4) << (j + 0); |
| 2202 | qh |= ((xi1 & 0x10u) >> 4) << (j + QK5_032/2); |
| 2203 | } |
| 2204 | memcpy(&y[ib].qh, &qh, sizeof(qh)); |
| 2205 | } |
| 2206 | } |
| 2207 | |
| 2208 | size_t quantize_q5_1(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2209 | if (!quant_weights) { |
| 2210 | quantize_row_q5_1_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2211 | return nrow * ggml_row_size(GGML_TYPE_Q5_1, n_per_row); |
| 2212 | } |
| 2213 | size_t row_size = ggml_row_size(GGML_TYPE_Q5_1, n_per_row); |
| 2214 | char * qrow = (char *)dst; |
| 2215 | for (int64_t row = 0; row < nrow; ++row) { |
| 2216 | quantize_row_q5_1_impl(src, (block_q5_1*)qrow, n_per_row, quant_weights); |
| 2217 | src += n_per_row; |
| 2218 | qrow += row_size; |
| 2219 | } |
| 2220 | return nrow * row_size; |
| 2221 | } |
| 2222 | |
| 2223 | size_t quantize_q8_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2224 | (void)quant_weights; // not used |
| 2225 | const size_t row_size = ggml_row_size(GGML_TYPE_Q8_0, n_per_row); |
| 2226 | quantize_row_q8_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2227 | return nrow * row_size; |
| 2228 | } |
| 2229 | |
| 2230 | size_t quantize_mxfp4(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2231 | GGML_UNUSED(quant_weights)(void)(quant_weights); |
| 2232 | quantize_row_mxfp4_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2233 | return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row); |
| 2234 | } |
| 2235 | |
| 2236 | size_t quantize_nvfp4(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2237 | GGML_UNUSED(quant_weights)(void)(quant_weights); |
| 2238 | quantize_row_nvfp4_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2239 | return nrow * ggml_row_size(GGML_TYPE_NVFP4, n_per_row); |
| 2240 | } |
| 2241 | |
| 2242 | // ====================== Ternary (de)-quantization (BitNet b1.58 and TriLMs) |
| 2243 | |
| 2244 | void quantize_row_tq1_0_ref(const float * GGML_RESTRICTrestrict x, block_tq1_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 2245 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2245, __extension__ __PRETTY_FUNCTION__); })); |
| 2246 | const int64_t nb = k / QK_K256; |
| 2247 | |
| 2248 | for (int64_t i = 0; i < nb; i++) { |
| 2249 | float amax = 0.0f; // absolute max |
| 2250 | |
| 2251 | for (int j = 0; j < QK_K256; j++) { |
| 2252 | const float v = x[j]; |
| 2253 | amax = MAX(amax, fabsf(v))((amax) > (fabsf(v)) ? (amax) : (fabsf(v))); |
| 2254 | } |
| 2255 | |
| 2256 | const float d = amax; |
| 2257 | const float id = d ? 1.0f/d : 0.0f; |
| 2258 | |
| 2259 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2260 | |
| 2261 | // 5 elements per byte, along 32 bytes |
| 2262 | for (size_t j = 0; j < sizeof(y->qs) - sizeof(y->qs) % 32; j += 32) { |
| 2263 | for (size_t m = 0; m < 32; ++m) { |
| 2264 | uint8_t q = 0; |
| 2265 | for (size_t n = 0; n < 5; ++n) { |
| 2266 | int xi = lroundf(x[m + n*32] * id) + 1; // -1, 0, 1 -> 0, 1, 2 |
| 2267 | q *= 3; |
| 2268 | q += xi; |
| 2269 | } |
| 2270 | // ceiling division (243 == pow(3, 5)) |
| 2271 | q = ((uint16_t)q * 256 + (243 - 1)) / 243; |
| 2272 | y[i].qs[j + m] = q; |
| 2273 | } |
| 2274 | x += 5*32; |
| 2275 | } |
| 2276 | // along 16 bytes |
| 2277 | for (size_t j = sizeof(y->qs) - sizeof(y->qs) % 32; j < sizeof(y->qs); j += 16) { |
| 2278 | for (size_t m = 0; m < 16; ++m) { |
| 2279 | uint8_t q = 0; |
| 2280 | for (size_t n = 0; n < 5; ++n) { |
| 2281 | int xi = lroundf(x[m + n*16] * id) + 1; // -1, 0, 1 -> 0, 1, 2 |
| 2282 | q *= 3; |
| 2283 | q += xi; |
| 2284 | } |
| 2285 | // ceiling division (243 == pow(3, 5)) |
| 2286 | q = ((uint16_t)q * 256 + (243 - 1)) / 243; |
| 2287 | y[i].qs[j + m] = q; |
| 2288 | } |
| 2289 | x += 5*16; |
| 2290 | } |
| 2291 | // 4 elements per byte |
| 2292 | for (size_t j = 0; j < sizeof(y->qh); ++j) { |
| 2293 | uint8_t q = 0; |
| 2294 | for (size_t m = 0; m < 4; ++m) { |
| 2295 | // -1, 0, 1 -> 0, 1, 2 |
| 2296 | int xi = lroundf(x[j + m*sizeof(y->qh)] * id) + 1; |
| 2297 | q *= 3; |
| 2298 | q += xi; |
| 2299 | } |
| 2300 | // shift the first value to the most significant trit |
| 2301 | q *= 3; |
| 2302 | // ceiling division (243 == pow(3, 5)) |
| 2303 | q = ((uint16_t)q * 256 + (243 - 1)) / 243; |
| 2304 | y[i].qh[j] = q; |
| 2305 | } |
| 2306 | x += 4*sizeof(y->qh); |
| 2307 | } |
| 2308 | } |
| 2309 | |
| 2310 | void quantize_row_tq2_0_ref(const float * GGML_RESTRICTrestrict x, block_tq2_0 * GGML_RESTRICTrestrict y, int64_t k) { |
| 2311 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2311, __extension__ __PRETTY_FUNCTION__); })); |
| 2312 | const int64_t nb = k / QK_K256; |
| 2313 | |
| 2314 | for (int64_t i = 0; i < nb; i++) { |
| 2315 | float amax = 0.0f; // absolute max |
| 2316 | |
| 2317 | for (int j = 0; j < QK_K256; j++) { |
| 2318 | const float v = x[j]; |
| 2319 | amax = MAX(amax, fabsf(v))((amax) > (fabsf(v)) ? (amax) : (fabsf(v))); |
| 2320 | } |
| 2321 | |
| 2322 | const float d = amax; |
| 2323 | const float id = d ? 1.0f/d : 0.0f; |
| 2324 | |
| 2325 | y[i].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 2326 | |
| 2327 | for (size_t j = 0; j < sizeof(y->qs); j += 32) { |
| 2328 | for (size_t m = 0; m < 32; ++m) { |
| 2329 | uint8_t q = 0; |
| 2330 | for (size_t n = 0; n < 4; ++n) { |
| 2331 | // -1, 0, 1 -> 0, 1, 2 |
| 2332 | int xi = lroundf(x[m + n*32] * id) + 1; |
| 2333 | q += (xi & 3) << (2*n); |
| 2334 | } |
| 2335 | y[i].qs[j + m] = q; |
| 2336 | } |
| 2337 | x += 4*32; |
| 2338 | } |
| 2339 | } |
| 2340 | } |
| 2341 | |
| 2342 | size_t quantize_tq1_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2343 | (void)quant_weights; // not used |
| 2344 | const size_t row_size = ggml_row_size(GGML_TYPE_TQ1_0, n_per_row); |
| 2345 | quantize_row_tq1_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2346 | return nrow * row_size; |
| 2347 | } |
| 2348 | |
| 2349 | size_t quantize_tq2_0(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 2350 | (void)quant_weights; // not used |
| 2351 | const size_t row_size = ggml_row_size(GGML_TYPE_TQ2_0, n_per_row); |
| 2352 | quantize_row_tq2_0_ref(src, dst, (int64_t)nrow*n_per_row); |
| 2353 | return nrow * row_size; |
| 2354 | } |
| 2355 | |
| 2356 | void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2357 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2357, __extension__ __PRETTY_FUNCTION__); })); |
| 2358 | const int64_t nb = k / QK_K256; |
| 2359 | |
| 2360 | const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; |
| 2361 | |
| 2362 | for (int64_t i = 0; i < nb; ++i) { |
| 2363 | |
| 2364 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2365 | |
| 2366 | for (size_t j = 0; j < sizeof(x->qs) - sizeof(x->qs) % 32; j += 32) { |
| 2367 | for (size_t n = 0; n < 5; ++n) { |
| 2368 | for (size_t m = 0; m < 32; ++m) { |
| 2369 | uint8_t q = x[i].qs[j + m] * pow3[n]; |
| 2370 | int16_t xi = ((uint16_t) q * 3) >> 8; |
| 2371 | *y++ = (float) (xi - 1) * d; |
| 2372 | } |
| 2373 | } |
| 2374 | } |
| 2375 | for (size_t j = sizeof(x->qs) - sizeof(x->qs) % 32; j < sizeof(x->qs); j += 16) { |
| 2376 | for (size_t n = 0; n < 5; ++n) { |
| 2377 | for (size_t m = 0; m < 16; ++m) { |
| 2378 | uint8_t q = x[i].qs[j + m] * pow3[n]; |
| 2379 | int16_t xi = ((uint16_t) q * 3) >> 8; |
| 2380 | *y++ = (float) (xi - 1) * d; |
| 2381 | } |
| 2382 | } |
| 2383 | } |
| 2384 | |
| 2385 | for (size_t n = 0; n < 4; ++n) { |
| 2386 | for (size_t j = 0; j < sizeof(x->qh); ++j) { |
| 2387 | uint8_t q = x[i].qh[j] * pow3[n]; |
| 2388 | int16_t xi = ((uint16_t) q * 3) >> 8; |
| 2389 | *y++ = (float) (xi - 1) * d; |
| 2390 | } |
| 2391 | } |
| 2392 | } |
| 2393 | } |
| 2394 | |
| 2395 | void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2396 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2396, __extension__ __PRETTY_FUNCTION__); })); |
| 2397 | const int64_t nb = k / QK_K256; |
| 2398 | |
| 2399 | for (int64_t i = 0; i < nb; ++i) { |
| 2400 | |
| 2401 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2402 | |
| 2403 | for (size_t j = 0; j < sizeof(x->qs); j += 32) { |
| 2404 | for (size_t l = 0; l < 4; ++l) { |
| 2405 | for (size_t m = 0; m < 32; ++m) { |
| 2406 | int8_t q = (x[i].qs[j + m] >> (l*2)) & 3; |
| 2407 | *y++ = (float) (q - 1) * d; |
| 2408 | } |
| 2409 | } |
| 2410 | } |
| 2411 | } |
| 2412 | } |
| 2413 | |
| 2414 | // ====================== "True" 2-bit (de)-quantization |
| 2415 | |
| 2416 | void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2417 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2417, __extension__ __PRETTY_FUNCTION__); })); |
| 2418 | const int64_t nb = k / QK_K256; |
| 2419 | |
| 2420 | uint32_t aux32[2]; |
| 2421 | const uint8_t * aux8 = (const uint8_t *)aux32; |
| 2422 | |
| 2423 | for (int i = 0; i < nb; i++) { |
| 2424 | |
| 2425 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2426 | |
| 2427 | for (int ib32 = 0; ib32 < QK_K256/32; ++ib32) { |
| 2428 | memcpy(aux32, x[i].qs + 4*ib32, 2*sizeof(uint32_t)); |
| 2429 | const float db = d * (0.5f + (aux32[1] >> 28)) * 0.25f; |
| 2430 | for (int l = 0; l < 4; ++l) { |
| 2431 | const uint8_t * grid = (const uint8_t *)(iq2xxs_grid + aux8[l]); |
| 2432 | const uint8_t signs = ksigns_iq2xs[(aux32[1] >> 7*l) & 127]; |
| 2433 | for (int j = 0; j < 8; ++j) { |
| 2434 | y[j] = db * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); |
| 2435 | } |
| 2436 | y += 8; |
| 2437 | } |
| 2438 | } |
| 2439 | } |
| 2440 | } |
| 2441 | |
| 2442 | // ====================== 2.3125 bpw (de)-quantization |
| 2443 | |
| 2444 | void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2445 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2445, __extension__ __PRETTY_FUNCTION__); })); |
| 2446 | const int64_t nb = k / QK_K256; |
| 2447 | |
| 2448 | float db[2]; |
| 2449 | |
| 2450 | for (int i = 0; i < nb; i++) { |
| 2451 | |
| 2452 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2453 | |
| 2454 | for (int ib32 = 0; ib32 < QK_K256/32; ++ib32) { |
| 2455 | db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f; |
| 2456 | db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f; |
| 2457 | for (int l = 0; l < 4; ++l) { |
| 2458 | const uint8_t * grid = (const uint8_t *)(iq2xs_grid + (x[i].qs[4*ib32 + l] & 511)); |
| 2459 | const uint8_t signs = ksigns_iq2xs[x[i].qs[4*ib32 + l] >> 9]; |
| 2460 | for (int j = 0; j < 8; ++j) { |
| 2461 | y[j] = db[l/2] * grid[j] * (signs & kmask_iq2xs[j] ? -1.f : 1.f); |
| 2462 | } |
| 2463 | y += 8; |
| 2464 | } |
| 2465 | } |
| 2466 | } |
| 2467 | } |
| 2468 | |
| 2469 | // ====================== 2.5625 bpw (de)-quantization |
| 2470 | |
| 2471 | void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2472 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2472, __extension__ __PRETTY_FUNCTION__); })); |
| 2473 | const int64_t nb = k / QK_K256; |
| 2474 | |
| 2475 | float db[2]; |
| 2476 | |
| 2477 | for (int i = 0; i < nb; i++) { |
| 2478 | |
| 2479 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2480 | const uint8_t * qs = x[i].qs; |
| 2481 | const uint8_t * qh = x[i].qh; |
| 2482 | const uint8_t * signs = qs + QK_K256/8; |
| 2483 | |
| 2484 | for (int ib32 = 0; ib32 < QK_K256/32; ++ib32) { |
| 2485 | db[0] = d * (0.5f + (x[i].scales[ib32] & 0xf)) * 0.25f; |
| 2486 | db[1] = d * (0.5f + (x[i].scales[ib32] >> 4)) * 0.25f; |
| 2487 | for (int l = 0; l < 4; ++l) { |
| 2488 | const float dl = db[l/2]; |
| 2489 | const uint8_t * grid = (const uint8_t *)(iq2s_grid + (qs[l] | (qh[ib32] << (8-2*l) & 0x300))); |
| 2490 | for (int j = 0; j < 8; ++j) { |
| 2491 | y[j] = dl * grid[j] * (signs[l] & kmask_iq2xs[j] ? -1.f : 1.f); |
| 2492 | } |
| 2493 | y += 8; |
| 2494 | } |
| 2495 | qs += 4; |
| 2496 | signs += 4; |
| 2497 | } |
| 2498 | } |
| 2499 | } |
| 2500 | |
| 2501 | // ====================== 3.0625 bpw (de)-quantization |
| 2502 | |
| 2503 | void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2504 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2504, __extension__ __PRETTY_FUNCTION__); })); |
| 2505 | const int64_t nb = k / QK_K256; |
| 2506 | |
| 2507 | uint32_t aux32; |
| 2508 | |
| 2509 | for (int i = 0; i < nb; i++) { |
| 2510 | |
| 2511 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2512 | const uint8_t * qs = x[i].qs; |
| 2513 | const uint8_t * scales_and_signs = qs + QK_K256/4; |
| 2514 | |
| 2515 | for (int ib32 = 0; ib32 < QK_K256/32; ++ib32) { |
| 2516 | memcpy(&aux32, scales_and_signs + 4*ib32, sizeof(uint32_t)); |
| 2517 | const float db = d * (0.5f + (aux32 >> 28)) * 0.5f; |
| 2518 | for (int l = 0; l < 4; ++l) { |
| 2519 | const uint8_t signs = ksigns_iq2xs[(aux32 >> 7*l) & 127]; |
| 2520 | const uint8_t * grid1 = (const uint8_t *)(iq3xxs_grid + qs[2*l+0]); |
| 2521 | const uint8_t * grid2 = (const uint8_t *)(iq3xxs_grid + qs[2*l+1]); |
| 2522 | for (int j = 0; j < 4; ++j) { |
| 2523 | y[j+0] = db * grid1[j] * (signs & kmask_iq2xs[j+0] ? -1.f : 1.f); |
| 2524 | y[j+4] = db * grid2[j] * (signs & kmask_iq2xs[j+4] ? -1.f : 1.f); |
| 2525 | } |
| 2526 | y += 8; |
| 2527 | } |
| 2528 | qs += 8; |
| 2529 | } |
| 2530 | } |
| 2531 | } |
| 2532 | |
| 2533 | // ====================== 3.3125 bpw (de)-quantization |
| 2534 | |
| 2535 | void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2536 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2536, __extension__ __PRETTY_FUNCTION__); })); |
| 2537 | const int64_t nb = k / QK_K256; |
| 2538 | |
| 2539 | for (int i = 0; i < nb; i++) { |
| 2540 | |
| 2541 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2542 | const uint8_t * qs = x[i].qs; |
| 2543 | const uint8_t * qh = x[i].qh; |
| 2544 | const uint8_t * signs = x[i].signs; |
| 2545 | |
| 2546 | for (int ib32 = 0; ib32 < QK_K256/32; ib32 += 2) { |
| 2547 | const float db1 = d * (1 + 2*(x[i].scales[ib32/2] & 0xf)); |
| 2548 | const float db2 = d * (1 + 2*(x[i].scales[ib32/2] >> 4)); |
| 2549 | for (int l = 0; l < 4; ++l) { |
| 2550 | const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[0] << (8-2*l)) & 256))); |
| 2551 | const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[0] << (7-2*l)) & 256))); |
| 2552 | for (int j = 0; j < 4; ++j) { |
| 2553 | y[j+0] = db1 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); |
| 2554 | y[j+4] = db1 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); |
| 2555 | } |
| 2556 | y += 8; |
| 2557 | } |
| 2558 | qs += 8; |
| 2559 | signs += 4; |
| 2560 | for (int l = 0; l < 4; ++l) { |
| 2561 | const uint8_t * grid1 = (const uint8_t *)(iq3s_grid + (qs[2*l+0] | ((qh[1] << (8-2*l)) & 256))); |
| 2562 | const uint8_t * grid2 = (const uint8_t *)(iq3s_grid + (qs[2*l+1] | ((qh[1] << (7-2*l)) & 256))); |
| 2563 | for (int j = 0; j < 4; ++j) { |
| 2564 | y[j+0] = db2 * grid1[j] * (signs[l] & kmask_iq2xs[j+0] ? -1.f : 1.f); |
| 2565 | y[j+4] = db2 * grid2[j] * (signs[l] & kmask_iq2xs[j+4] ? -1.f : 1.f); |
| 2566 | } |
| 2567 | y += 8; |
| 2568 | } |
| 2569 | qh += 2; |
| 2570 | qs += 8; |
| 2571 | signs += 4; |
| 2572 | } |
| 2573 | } |
| 2574 | } |
| 2575 | |
| 2576 | // ====================== 1.5625 bpw (de)-quantization |
| 2577 | |
| 2578 | void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2579 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2579, __extension__ __PRETTY_FUNCTION__); })); |
| 2580 | const int64_t nb = k / QK_K256; |
| 2581 | |
| 2582 | for (int i = 0; i < nb; i++) { |
| 2583 | |
| 2584 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2585 | const uint8_t * qs = x[i].qs; |
| 2586 | const uint16_t * qh = x[i].qh; |
| 2587 | |
| 2588 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 2589 | const float dl = d * (2*((qh[ib] >> 12) & 7) + 1); |
| 2590 | const float delta = qh[ib] & 0x8000 ? -IQ1S_DELTA0.125f : IQ1S_DELTA0.125f; |
| 2591 | for (int l = 0; l < 4; ++l) { |
| 2592 | const int8_t * grid = (const int8_t *)(iq1s_grid + (qs[l] | (((qh[ib] >> 3*l) & 7) << 8))); |
| 2593 | for (int j = 0; j < 8; ++j) { |
| 2594 | y[j] = dl * (grid[j] + delta); |
| 2595 | } |
| 2596 | y += 8; |
| 2597 | } |
| 2598 | qs += 4; |
| 2599 | } |
| 2600 | } |
| 2601 | } |
| 2602 | |
| 2603 | void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2604 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2604, __extension__ __PRETTY_FUNCTION__); })); |
| 2605 | const int64_t nb = k / QK_K256; |
| 2606 | |
| 2607 | float delta[4]; |
| 2608 | uint16_t idx[4]; |
| 2609 | |
| 2610 | iq1m_scale_t scale; |
| 2611 | |
| 2612 | for (int i = 0; i < nb; i++) { |
| 2613 | |
| 2614 | const uint16_t * sc = (const uint16_t *)x[i].scales; |
| 2615 | scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); |
| 2616 | const float d = GGML_FP16_TO_FP32(scale.f16)ggml_compute_fp16_to_fp32(scale.f16); |
| 2617 | |
| 2618 | const uint8_t * qs = x[i].qs; |
| 2619 | const uint8_t * qh = x[i].qh; |
| 2620 | |
| 2621 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 2622 | const float dl1 = d * (2*((sc[ib/2] >> (6*(ib%2)+0)) & 0x7) + 1); |
| 2623 | const float dl2 = d * (2*((sc[ib/2] >> (6*(ib%2)+3)) & 0x7) + 1); |
| 2624 | |
| 2625 | idx[0] = qs[0] | ((qh[0] << 8) & 0x700); |
| 2626 | idx[1] = qs[1] | ((qh[0] << 4) & 0x700); |
| 2627 | idx[2] = qs[2] | ((qh[1] << 8) & 0x700); |
| 2628 | idx[3] = qs[3] | ((qh[1] << 4) & 0x700); |
| 2629 | delta[0] = qh[0] & 0x08 ? -IQ1S_DELTA0.125f : IQ1S_DELTA0.125f; |
| 2630 | delta[1] = qh[0] & 0x80 ? -IQ1S_DELTA0.125f : IQ1S_DELTA0.125f; |
| 2631 | delta[2] = qh[1] & 0x08 ? -IQ1S_DELTA0.125f : IQ1S_DELTA0.125f; |
| 2632 | delta[3] = qh[1] & 0x80 ? -IQ1S_DELTA0.125f : IQ1S_DELTA0.125f; |
| 2633 | for (int l = 0; l < 2; ++l) { |
| 2634 | const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); |
| 2635 | for (int j = 0; j < 8; ++j) { |
| 2636 | y[j] = dl1 * (grid[j] + delta[l]); |
| 2637 | } |
| 2638 | y += 8; |
| 2639 | } |
| 2640 | for (int l = 2; l < 4; ++l) { |
| 2641 | const int8_t * grid = (const int8_t *)(iq1s_grid + idx[l]); |
| 2642 | for (int j = 0; j < 8; ++j) { |
| 2643 | y[j] = dl2 * (grid[j] + delta[l]); |
| 2644 | } |
| 2645 | y += 8; |
| 2646 | } |
| 2647 | qs += 4; |
| 2648 | qh += 2; |
| 2649 | } |
| 2650 | } |
| 2651 | } |
| 2652 | |
| 2653 | void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2654 | assert(k % QK4_NL == 0)((void) sizeof (__assert_single_arg (k % 32 == 0)), __extension__ ({ if (k % 32 == 0) ; else __assert_fail ("k % QK4_NL == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2654, __extension__ __PRETTY_FUNCTION__); })); |
| 2655 | const int64_t nb = k / QK4_NL32; |
| 2656 | |
| 2657 | for (int i = 0; i < nb; i++) { |
| 2658 | |
| 2659 | const uint8_t * qs = x[i].qs; |
| 2660 | |
| 2661 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2662 | for (int j = 0; j < QK4_NL32/2; ++j) { |
| 2663 | y[j+ 0] = d * kvalues_iq4nl[qs[j] & 0xf]; |
| 2664 | y[j+QK4_NL32/2] = d * kvalues_iq4nl[qs[j] >> 4]; |
| 2665 | } |
| 2666 | y += QK4_NL32; |
| 2667 | qs += QK4_NL32/2; |
| 2668 | } |
| 2669 | } |
| 2670 | |
| 2671 | void dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2672 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2672, __extension__ __PRETTY_FUNCTION__); })); |
| 2673 | const int64_t nb = k / QK_K256; |
| 2674 | |
| 2675 | for (int i = 0; i < nb; i++) { |
| 2676 | |
| 2677 | const uint8_t * qs = x[i].qs; |
| 2678 | |
| 2679 | const float d = GGML_FP16_TO_FP32(x[i].d)ggml_compute_fp16_to_fp32(x[i].d); |
| 2680 | |
| 2681 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 2682 | const int ls = ((x[i].scales_l[ib/2] >> 4*(ib%2)) & 0xf) | (((x[i].scales_h >> 2*ib) & 3) << 4); |
| 2683 | const float dl = d * (ls - 32); |
| 2684 | for (int j = 0; j < 16; ++j) { |
| 2685 | y[j+ 0] = dl * kvalues_iq4nl[qs[j] & 0xf]; |
| 2686 | y[j+16] = dl * kvalues_iq4nl[qs[j] >> 4]; |
| 2687 | } |
| 2688 | y += 32; |
| 2689 | qs += 16; |
| 2690 | } |
| 2691 | } |
| 2692 | } |
| 2693 | |
| 2694 | //===================================== Q8_K ============================================== |
| 2695 | |
| 2696 | void quantize_row_q8_K_ref(const float * GGML_RESTRICTrestrict x, block_q8_K * GGML_RESTRICTrestrict y, int64_t k) { |
| 2697 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2697, __extension__ __PRETTY_FUNCTION__); })); |
| 2698 | const int64_t nb = k / QK_K256; |
| 2699 | |
| 2700 | for (int i = 0; i < nb; i++) { |
| 2701 | |
| 2702 | float max = 0; |
| 2703 | float amax = 0; |
| 2704 | for (int j = 0; j < QK_K256; ++j) { |
| 2705 | float ax = fabsf(x[j]); |
| 2706 | if (ax > amax) { |
| 2707 | amax = ax; max = x[j]; |
| 2708 | } |
| 2709 | } |
| 2710 | if (!amax) { |
| 2711 | y[i].d = 0; |
| 2712 | memset(y[i].qs, 0, QK_K256); |
| 2713 | x += QK_K256; |
| 2714 | continue; |
| 2715 | } |
| 2716 | //const float iscale = -128.f/max; |
| 2717 | // We need this change for IQ2_XXS, else the AVX implementation becomes very awkward |
| 2718 | const float iscale = -127.f/max; |
| 2719 | for (int j = 0; j < QK_K256; ++j) { |
| 2720 | int v = nearest_int(iscale*x[j]); |
| 2721 | y[i].qs[j] = MIN(127, v)((127) < (v) ? (127) : (v)); |
| 2722 | } |
| 2723 | for (int j = 0; j < QK_K256/16; ++j) { |
| 2724 | int sum = 0; |
| 2725 | for (int ii = 0; ii < 16; ++ii) { |
| 2726 | sum += y[i].qs[j*16 + ii]; |
| 2727 | } |
| 2728 | y[i].bsums[j] = sum; |
| 2729 | } |
| 2730 | y[i].d = 1/iscale; |
| 2731 | x += QK_K256; |
| 2732 | } |
| 2733 | } |
| 2734 | |
| 2735 | void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICTrestrict x, float * GGML_RESTRICTrestrict y, int64_t k) { |
| 2736 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2736, __extension__ __PRETTY_FUNCTION__); })); |
| 2737 | const int64_t nb = k / QK_K256; |
| 2738 | |
| 2739 | for (int i = 0; i < nb; i++) { |
| 2740 | for (int j = 0; j < QK_K256; ++j) { |
| 2741 | *y++ = x[i].d * x[i].qs[j]; |
| 2742 | } |
| 2743 | } |
| 2744 | } |
| 2745 | |
| 2746 | // ================================ IQ2 quantization ============================================= |
| 2747 | |
| 2748 | typedef struct { |
| 2749 | uint64_t * grid; |
| 2750 | int * map; |
| 2751 | uint16_t * neighbours; |
| 2752 | } iq2_entry_t; |
| 2753 | |
| 2754 | static iq2_entry_t iq2_data[4] = { |
| 2755 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 2756 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 2757 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 2758 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 2759 | }; |
| 2760 | |
| 2761 | static inline int iq2_data_index(enum ggml_type type) { |
| 2762 | GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)if (!(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2762, "GGML_ASSERT(%s) failed", "type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S" ); |
| 2763 | return type == GGML_TYPE_IQ2_XXS ? 0 : |
| 2764 | type == GGML_TYPE_IQ2_XS ? 1 : |
| 2765 | type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 2 : 3; |
| 2766 | } |
| 2767 | |
| 2768 | static inline int iq2_grid_size(enum ggml_type type) { |
| 2769 | GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)if (!(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 2769, "GGML_ASSERT(%s) failed", "type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S" ); |
| 2770 | return type == GGML_TYPE_IQ2_XXS ? 256 : |
| 2771 | type == GGML_TYPE_IQ2_XS ? 512 : |
| 2772 | type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? NGRID_IQ1S2048 : 1024; |
| 2773 | } |
| 2774 | |
| 2775 | static int iq2_compare_func(const void * left, const void * right) { |
| 2776 | const int * l = (const int *)left; |
| 2777 | const int * r = (const int *)right; |
| 2778 | return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0; |
| 2779 | } |
| 2780 | |
| 2781 | void iq2xs_init_impl(enum ggml_type type) { |
| 2782 | const int gindex = iq2_data_index(type); |
| 2783 | const int grid_size = iq2_grid_size(type); |
| 2784 | if (iq2_data[gindex].grid) { |
| 2785 | return; |
| 2786 | } |
| 2787 | static const uint16_t kgrid_2bit_256[256] = { |
| 2788 | 0, 2, 5, 8, 10, 17, 20, 32, 34, 40, 42, 65, 68, 80, 88, 97, |
| 2789 | 100, 128, 130, 138, 162, 257, 260, 272, 277, 320, 388, 408, 512, 514, 546, 642, |
| 2790 | 1025, 1028, 1040, 1057, 1060, 1088, 1090, 1096, 1120, 1153, 1156, 1168, 1188, 1280, 1282, 1288, |
| 2791 | 1312, 1350, 1385, 1408, 1425, 1545, 1552, 1600, 1668, 1700, 2048, 2053, 2056, 2068, 2088, 2113, |
| 2792 | 2116, 2128, 2130, 2184, 2308, 2368, 2562, 2580, 4097, 4100, 4112, 4129, 4160, 4192, 4228, 4240, |
| 2793 | 4245, 4352, 4360, 4384, 4432, 4442, 4480, 4644, 4677, 5120, 5128, 5152, 5157, 5193, 5248, 5400, |
| 2794 | 5474, 5632, 5654, 6145, 6148, 6160, 6208, 6273, 6400, 6405, 6560, 6737, 8192, 8194, 8202, 8260, |
| 2795 | 8289, 8320, 8322, 8489, 8520, 8704, 8706, 9217, 9220, 9232, 9280, 9302, 9472, 9537, 9572, 9872, |
| 2796 | 10248, 10272, 10388, 10820, 16385, 16388, 16400, 16408, 16417, 16420, 16448, 16456, 16470, 16480, 16513, 16516, |
| 2797 | 16528, 16640, 16672, 16737, 16768, 16773, 16897, 16912, 16968, 16982, 17000, 17408, 17416, 17440, 17536, 17561, |
| 2798 | 17682, 17700, 17920, 18433, 18436, 18448, 18496, 18501, 18688, 18776, 18785, 18818, 19013, 19088, 20480, 20488, |
| 2799 | 20497, 20505, 20512, 20608, 20616, 20740, 20802, 20900, 21137, 21648, 21650, 21770, 22017, 22100, 22528, 22545, |
| 2800 | 22553, 22628, 22848, 23048, 24580, 24592, 24640, 24680, 24832, 24917, 25112, 25184, 25600, 25605, 25872, 25874, |
| 2801 | 25988, 26690, 32768, 32770, 32778, 32833, 32898, 33028, 33048, 33088, 33297, 33793, 33796, 33808, 33813, 33856, |
| 2802 | 33888, 34048, 34118, 34196, 34313, 34368, 34400, 34818, 35076, 35345, 36868, 36880, 36900, 36928, 37025, 37142, |
| 2803 | 37248, 37445, 37888, 37922, 37956, 38225, 39041, 39200, 40962, 41040, 41093, 41225, 41472, 42008, 43088, 43268, |
| 2804 | }; |
| 2805 | static const uint16_t kgrid_2bit_512[512] = { |
| 2806 | 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, |
| 2807 | 73, 80, 82, 85, 88, 97, 100, 128, 130, 133, 136, 145, 148, 153, 160, 257, |
| 2808 | 260, 262, 265, 272, 274, 277, 280, 282, 289, 292, 320, 322, 325, 328, 337, 340, |
| 2809 | 352, 360, 385, 388, 400, 512, 514, 517, 520, 529, 532, 544, 577, 580, 592, 597, |
| 2810 | 640, 650, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1088, 1090, 1093, 1096, |
| 2811 | 1105, 1108, 1110, 1120, 1153, 1156, 1168, 1280, 1282, 1285, 1288, 1297, 1300, 1312, 1345, 1348, |
| 2812 | 1360, 1377, 1408, 1537, 1540, 1552, 1574, 1600, 1602, 1668, 2048, 2050, 2053, 2056, 2058, 2065, |
| 2813 | 2068, 2080, 2085, 2113, 2116, 2128, 2136, 2176, 2208, 2218, 2305, 2308, 2320, 2368, 2433, 2441, |
| 2814 | 2560, 2592, 2600, 2710, 2720, 4097, 4100, 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4160, |
| 2815 | 4162, 4165, 4168, 4177, 4180, 4192, 4202, 4225, 4228, 4240, 4352, 4354, 4357, 4360, 4369, 4372, |
| 2816 | 4384, 4417, 4420, 4432, 4480, 4500, 4502, 4609, 4612, 4614, 4624, 4672, 4704, 5120, 5122, 5125, |
| 2817 | 5128, 5137, 5140, 5152, 5185, 5188, 5193, 5200, 5220, 5248, 5377, 5380, 5392, 5440, 5632, 5652, |
| 2818 | 5705, 6145, 6148, 6160, 6162, 6208, 6228, 6278, 6400, 6405, 6502, 6737, 6825, 8192, 8194, 8197, |
| 2819 | 8200, 8202, 8209, 8212, 8224, 8257, 8260, 8272, 8320, 8352, 8449, 8452, 8464, 8512, 8520, 8549, |
| 2820 | 8704, 8738, 8832, 8872, 9217, 9220, 9232, 9257, 9280, 9472, 9537, 9554, 9625, 9729, 9754, 9894, |
| 2821 | 10240, 10248, 10250, 10272, 10325, 10376, 10402, 10600, 10640, 10760, 10784, 10882, 10888, 10890, 16385, 16388, |
| 2822 | 16390, 16393, 16400, 16402, 16405, 16408, 16417, 16420, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16480, |
| 2823 | 16485, 16513, 16516, 16528, 16640, 16642, 16645, 16648, 16657, 16660, 16672, 16705, 16708, 16720, 16768, 16773, |
| 2824 | 16802, 16897, 16900, 16912, 16914, 16937, 16960, 17408, 17410, 17413, 17416, 17425, 17428, 17433, 17440, 17473, |
| 2825 | 17476, 17488, 17536, 17556, 17665, 17668, 17680, 17700, 17728, 17818, 17920, 17930, 17988, 18000, 18433, 18436, |
| 2826 | 18448, 18496, 18501, 18516, 18530, 18688, 18705, 18756, 18768, 18793, 18948, 20480, 20482, 20485, 20488, 20497, |
| 2827 | 20500, 20512, 20520, 20545, 20548, 20560, 20608, 20737, 20740, 20752, 20757, 20800, 20802, 20992, 21060, 21162, |
| 2828 | 21505, 21508, 21520, 21537, 21568, 21600, 21633, 21665, 21760, 21768, 21888, 21896, 22049, 22120, 22177, 22528, |
| 2829 | 22548, 22593, 22608, 22681, 22810, 22848, 22850, 23173, 24577, 24580, 24592, 24640, 24660, 24674, 24710, 24745, |
| 2830 | 24832, 25124, 25162, 25234, 25600, 25622, 25872, 25920, 25925, 26020, 26625, 26730, 26917, 27142, 27220, 27234, |
| 2831 | 32768, 32770, 32773, 32776, 32785, 32788, 32800, 32810, 32833, 32836, 32848, 32896, 32898, 32936, 32938, 33025, |
| 2832 | 33028, 33030, 33040, 33088, 33105, 33113, 33280, 33312, 33408, 33410, 33440, 33448, 33793, 33796, 33808, 33810, |
| 2833 | 33813, 33856, 33888, 33929, 34048, 34116, 34213, 34328, 34410, 34816, 34824, 34853, 34906, 34944, 34946, 34984, |
| 2834 | 35078, 35362, 35456, 35464, 35478, 35496, 36865, 36868, 36880, 36928, 36950, 36996, 37120, 37154, 37220, 37462, |
| 2835 | 37513, 37888, 37893, 37956, 37968, 37976, 38185, 38288, 38290, 38465, 38993, 39078, 39241, 39445, 39520, 40960, |
| 2836 | 40962, 40968, 40970, 40992, 41002, 41120, 41297, 41305, 41382, 41472, 41474, 41480, 41514, 41600, 41632, 42048, |
| 2837 | 42133, 42597, 42648, 43018, 43040, 43042, 43048, 43168, 43176, 43268, 43396, 43398, 43560, 43562, 43665, 43690, |
| 2838 | }; |
| 2839 | static const uint16_t kgrid_1bit_2048[NGRID_IQ1S2048] = { |
| 2840 | 0, 2, 5, 8, 10, 17, 21, 32, 34, 40, 42, 69, 81, 84, 86, 101, |
| 2841 | 128, 130, 136, 138, 149, 160, 162, 168, 170, 260, 261, 273, 276, 278, 281, 282, |
| 2842 | 293, 321, 326, 329, 338, 341, 346, 353, 356, 358, 360, 389, 401, 404, 406, 421, |
| 2843 | 512, 514, 520, 522, 533, 544, 546, 552, 554, 581, 593, 601, 612, 617, 640, 642, |
| 2844 | 648, 650, 657, 661, 665, 672, 674, 680, 682, 1041, 1044, 1046, 1061, 1089, 1097, 1109, |
| 2845 | 1114, 1124, 1125, 1169, 1177, 1189, 1281, 1284, 1285, 1286, 1301, 1304, 1306, 1321, 1344, 1349, |
| 2846 | 1354, 1360, 1361, 1364, 1365, 1366, 1369, 1376, 1378, 1381, 1384, 1386, 1409, 1425, 1429, 1432, |
| 2847 | 1434, 1441, 1444, 1445, 1446, 1449, 1556, 1561, 1601, 1604, 1616, 1618, 1621, 1624, 1632, 1633, |
| 2848 | 1638, 1641, 1669, 1681, 1684, 1689, 2048, 2050, 2056, 2058, 2069, 2080, 2082, 2088, 2090, 2117, |
| 2849 | 2129, 2134, 2149, 2176, 2178, 2184, 2186, 2197, 2208, 2210, 2216, 2218, 2309, 2321, 2324, 2329, |
| 2850 | 2340, 2341, 2369, 2384, 2385, 2389, 2401, 2404, 2409, 2449, 2452, 2454, 2457, 2469, 2560, 2562, |
| 2851 | 2568, 2570, 2581, 2592, 2594, 2600, 2602, 2629, 2641, 2649, 2657, 2661, 2688, 2690, 2693, 2696, |
| 2852 | 2698, 2709, 2720, 2722, 2728, 2730, 4112, 4113, 4116, 4121, 4132, 4133, 4161, 4164, 4176, 4181, |
| 2853 | 4184, 4193, 4196, 4197, 4201, 4241, 4244, 4246, 4257, 4261, 4353, 4356, 4358, 4361, 4368, 4370, |
| 2854 | 4373, 4376, 4385, 4388, 4393, 4421, 4426, 4432, 4433, 4434, 4436, 4437, 4438, 4441, 4448, 4453, |
| 2855 | 4484, 4498, 4501, 4513, 4516, 4625, 4628, 4630, 4645, 4672, 4678, 4681, 4690, 4693, 4696, 4698, |
| 2856 | 4708, 4710, 4741, 4753, 4756, 4758, 4773, 5121, 5126, 5129, 5140, 5141, 5144, 5145, 5153, 5158, |
| 2857 | 5185, 5189, 5190, 5192, 5194, 5201, 5204, 5205, 5206, 5209, 5218, 5221, 5224, 5252, 5257, 5264, |
| 2858 | 5268, 5269, 5272, 5273, 5274, 5281, 5284, 5285, 5289, 5378, 5381, 5386, 5393, 5396, 5397, 5398, |
| 2859 | 5401, 5408, 5410, 5413, 5416, 5418, 5441, 5444, 5445, 5446, 5457, 5458, 5460, 5461, 5462, 5465, |
| 2860 | 5466, 5473, 5476, 5477, 5478, 5481, 5504, 5506, 5508, 5509, 5512, 5514, 5520, 5521, 5524, 5525, |
| 2861 | 5526, 5529, 5530, 5536, 5538, 5541, 5633, 5636, 5637, 5638, 5653, 5654, 5656, 5658, 5665, 5670, |
| 2862 | 5696, 5698, 5700, 5701, 5704, 5706, 5713, 5717, 5718, 5720, 5721, 5729, 5732, 5733, 5736, 5737, |
| 2863 | 5738, 5766, 5770, 5778, 5781, 5796, 5801, 6161, 6166, 6181, 6209, 6212, 6214, 6217, 6224, 6229, |
| 2864 | 6232, 6234, 6240, 6241, 6244, 6246, 6249, 6277, 6289, 6292, 6309, 6416, 6418, 6421, 6426, 6433, |
| 2865 | 6437, 6466, 6468, 6469, 6472, 6481, 6484, 6485, 6486, 6489, 6490, 6496, 6501, 6506, 6537, 6545, |
| 2866 | 6546, 6549, 6552, 6561, 6566, 6569, 6665, 6678, 6692, 6694, 6724, 6726, 6729, 6736, 6738, 6741, |
| 2867 | 6744, 6753, 6758, 6761, 6789, 6801, 6806, 6810, 8192, 8194, 8200, 8202, 8213, 8224, 8226, 8229, |
| 2868 | 8232, 8234, 8261, 8273, 8281, 8289, 8293, 8320, 8322, 8328, 8330, 8341, 8352, 8354, 8357, 8360, |
| 2869 | 8362, 8453, 8465, 8468, 8473, 8485, 8514, 8516, 8521, 8533, 8536, 8538, 8545, 8548, 8549, 8550, |
| 2870 | 8581, 8592, 8598, 8601, 8613, 8705, 8712, 8714, 8721, 8725, 8736, 8738, 8744, 8746, 8773, 8785, |
| 2871 | 8790, 8793, 8805, 8833, 8840, 8842, 8849, 8853, 8864, 8866, 8872, 8874, 9221, 9236, 9238, 9241, |
| 2872 | 9253, 9284, 9285, 9286, 9289, 9298, 9301, 9304, 9306, 9318, 9349, 9361, 9364, 9369, 9377, 9381, |
| 2873 | 9481, 9493, 9505, 9513, 9536, 9541, 9544, 9553, 9556, 9557, 9561, 9570, 9573, 9576, 9609, 9616, |
| 2874 | 9620, 9621, 9624, 9626, 9633, 9636, 9638, 9641, 9733, 9744, 9746, 9753, 9765, 9793, 9801, 9813, |
| 2875 | 9824, 9825, 9833, 9860, 9862, 9872, 9882, 10240, 10242, 10248, 10250, 10261, 10272, 10274, 10280, 10282, |
| 2876 | 10309, 10321, 10324, 10341, 10368, 10370, 10376, 10378, 10400, 10402, 10408, 10410, 10505, 10513, 10516, 10521, |
| 2877 | 10533, 10566, 10569, 10578, 10581, 10593, 10596, 10598, 10601, 10629, 10640, 10646, 10649, 10660, 10661, 10752, |
| 2878 | 10754, 10760, 10762, 10784, 10786, 10792, 10794, 10821, 10833, 10838, 10841, 10853, 10880, 10882, 10888, 10890, |
| 2879 | 10901, 10912, 10914, 10920, 10922, 16389, 16401, 16406, 16421, 16457, 16466, 16469, 16472, 16474, 16481, 16484, |
| 2880 | 16486, 16532, 16537, 16545, 16550, 16640, 16641, 16644, 16646, 16649, 16658, 16661, 16662, 16664, 16666, 16673, |
| 2881 | 16678, 16681, 16709, 16712, 16714, 16721, 16724, 16725, 16726, 16729, 16730, 16741, 16744, 16746, 16769, 16772, |
| 2882 | 16774, 16784, 16786, 16789, 16800, 16801, 16802, 16901, 16913, 16916, 16918, 16933, 16961, 16978, 16981, 16986, |
| 2883 | 16996, 17001, 17033, 17044, 17061, 17409, 17429, 17433, 17449, 17477, 17480, 17482, 17489, 17492, 17493, 17494, |
| 2884 | 17505, 17506, 17509, 17512, 17514, 17537, 17542, 17545, 17552, 17554, 17557, 17568, 17569, 17577, 17665, 17666, |
| 2885 | 17669, 17674, 17681, 17684, 17685, 17686, 17689, 17696, 17701, 17706, 17729, 17732, 17733, 17734, 17737, 17744, |
| 2886 | 17745, 17748, 17749, 17750, 17752, 17753, 17761, 17764, 17765, 17766, 17769, 17794, 17796, 17797, 17800, 17809, |
| 2887 | 17812, 17813, 17814, 17817, 17818, 17829, 17832, 17834, 17921, 17925, 17929, 17940, 17941, 17944, 17946, 17953, |
| 2888 | 17956, 17961, 17984, 17986, 17989, 17992, 18000, 18001, 18002, 18005, 18006, 18009, 18018, 18021, 18024, 18049, |
| 2889 | 18053, 18058, 18068, 18069, 18081, 18084, 18086, 18437, 18449, 18453, 18458, 18469, 18498, 18505, 18512, 18517, |
| 2890 | 18520, 18529, 18532, 18534, 18537, 18565, 18577, 18580, 18582, 18585, 18597, 18689, 18693, 18694, 18698, 18704, |
| 2891 | 18708, 18709, 18712, 18721, 18724, 18726, 18752, 18757, 18762, 18769, 18770, 18772, 18773, 18774, 18777, 18784, |
| 2892 | 18786, 18789, 18790, 18794, 18822, 18825, 18834, 18837, 18838, 18840, 18849, 18852, 18854, 18857, 18966, 19012, |
| 2893 | 19014, 19017, 19029, 19032, 19034, 19044, 19049, 19092, 19109, 20481, 20484, 20485, 20486, 20489, 20498, 20501, |
| 2894 | 20506, 20513, 20516, 20521, 20544, 20549, 20552, 20561, 20564, 20565, 20566, 20569, 20581, 20584, 20614, 20617, |
| 2895 | 20629, 20632, 20640, 20641, 20646, 20649, 20741, 20744, 20745, 20746, 20753, 20756, 20757, 20758, 20760, 20761, |
| 2896 | 20768, 20773, 20774, 20776, 20778, 20801, 20804, 20805, 20806, 20809, 20816, 20817, 20818, 20820, 20821, 20822, |
| 2897 | 20824, 20825, 20826, 20833, 20836, 20837, 20838, 20841, 20866, 20869, 20881, 20884, 20885, 20886, 20889, 20896, |
| 2898 | 20901, 20906, 20993, 20998, 21010, 21013, 21018, 21025, 21028, 21058, 21061, 21066, 21073, 21076, 21077, 21078, |
| 2899 | 21081, 21090, 21093, 21125, 21136, 21138, 21141, 21145, 21146, 21156, 21508, 21509, 21521, 21524, 21525, 21526, |
| 2900 | 21528, 21529, 21537, 21541, 21544, 21546, 21569, 21572, 21573, 21574, 21577, 21578, 21584, 21585, 21588, 21589, |
| 2901 | 21590, 21592, 21593, 21594, 21601, 21602, 21604, 21605, 21606, 21609, 21632, 21640, 21642, 21649, 21652, 21653, |
| 2902 | 21654, 21657, 21665, 21668, 21669, 21674, 21761, 21762, 21764, 21765, 21766, 21769, 21776, 21777, 21778, 21780, |
| 2903 | 21781, 21782, 21785, 21786, 21793, 21796, 21797, 21798, 21801, 21824, 21825, 21826, 21828, 21829, 21830, 21832, |
| 2904 | 21833, 21840, 21841, 21842, 21844, 21845, 21846, 21848, 21849, 21850, 21856, 21857, 21860, 21861, 21862, 21864, |
| 2905 | 21865, 21866, 21889, 21892, 21893, 21897, 21898, 21904, 21905, 21908, 21909, 21910, 21912, 21913, 21921, 21924, |
| 2906 | 21925, 21926, 21929, 22016, 22017, 22018, 22020, 22022, 22024, 22025, 22033, 22036, 22037, 22040, 22041, 22048, |
| 2907 | 22049, 22050, 22052, 22053, 22054, 22056, 22057, 22081, 22085, 22086, 22088, 22089, 22090, 22096, 22097, 22098, |
| 2908 | 22100, 22101, 22102, 22104, 22105, 22106, 22113, 22116, 22117, 22121, 22146, 22149, 22150, 22152, 22153, 22154, |
| 2909 | 22161, 22165, 22170, 22178, 22181, 22182, 22184, 22185, 22532, 22533, 22534, 22537, 22544, 22549, 22552, 22561, |
| 2910 | 22570, 22597, 22600, 22602, 22609, 22612, 22613, 22614, 22616, 22617, 22624, 22626, 22628, 22629, 22658, 22665, |
| 2911 | 22672, 22674, 22677, 22680, 22689, 22697, 22785, 22786, 22789, 22794, 22801, 22804, 22805, 22806, 22809, 22821, |
| 2912 | 22849, 22852, 22853, 22854, 22857, 22864, 22865, 22866, 22868, 22869, 22870, 22872, 22873, 22874, 22881, 22884, |
| 2913 | 22885, 22886, 22889, 22913, 22917, 22921, 22929, 22932, 22933, 22934, 22936, 22937, 22949, 23044, 23048, 23061, |
| 2914 | 23066, 23072, 23077, 23078, 23081, 23109, 23112, 23113, 23121, 23125, 23126, 23128, 23129, 23138, 23141, 23144, |
| 2915 | 23146, 23169, 23178, 23186, 23189, 23190, 23192, 23194, 23201, 24581, 24596, 24598, 24601, 24613, 24644, 24656, |
| 2916 | 24661, 24662, 24664, 24666, 24673, 24676, 24678, 24681, 24705, 24726, 24741, 24833, 24836, 24838, 24841, 24850, |
| 2917 | 24853, 24865, 24866, 24870, 24873, 24901, 24905, 24913, 24917, 24918, 24921, 24933, 24934, 24938, 24964, 24970, |
| 2918 | 24978, 24981, 24993, 24998, 25001, 25105, 25110, 25113, 25152, 25153, 25158, 25173, 25174, 25176, 25184, 25221, |
| 2919 | 25233, 25238, 25253, 25617, 25618, 25621, 25622, 25626, 25633, 25638, 25641, 25664, 25666, 25669, 25672, 25674, |
| 2920 | 25681, 25684, 25685, 25686, 25689, 25690, 25696, 25698, 25701, 25732, 25733, 25737, 25744, 25746, 25748, 25749, |
| 2921 | 25750, 25752, 25754, 25761, 25764, 25769, 25861, 25864, 25866, 25873, 25877, 25878, 25881, 25924, 25925, 25926, |
| 2922 | 25929, 25936, 25937, 25940, 25941, 25942, 25945, 25953, 25956, 25957, 25958, 25961, 25990, 25993, 25994, 26001, |
| 2923 | 26005, 26006, 26009, 26010, 26018, 26021, 26022, 26024, 26114, 26121, 26133, 26144, 26150, 26152, 26153, 26176, |
| 2924 | 26181, 26184, 26186, 26193, 26196, 26197, 26198, 26200, 26202, 26208, 26213, 26216, 26240, 26242, 26245, 26250, |
| 2925 | 26260, 26262, 26264, 26265, 26272, 26276, 26278, 26282, 26646, 26649, 26661, 26689, 26706, 26709, 26714, 26721, |
| 2926 | 26729, 26757, 26769, 26776, 26790, 26881, 26884, 26896, 26901, 26913, 26916, 26918, 26921, 26944, 26945, 26949, |
| 2927 | 26950, 26952, 26961, 26964, 26965, 26966, 26969, 26976, 26981, 26986, 27010, 27012, 27018, 27029, 27041, 27044, |
| 2928 | 27045, 27049, 27153, 27158, 27160, 27201, 27204, 27209, 27216, 27221, 27224, 27226, 27236, 27237, 27241, 27270, |
| 2929 | 27284, 27288, 27290, 27302, 32768, 32770, 32776, 32778, 32800, 32802, 32808, 32810, 32837, 32848, 32849, 32852, |
| 2930 | 32854, 32857, 32869, 32896, 32898, 32904, 32906, 32917, 32928, 32930, 32936, 32938, 33029, 33041, 33044, 33046, |
| 2931 | 33049, 33061, 33089, 33092, 33097, 33104, 33106, 33109, 33110, 33112, 33113, 33124, 33126, 33129, 33157, 33161, |
| 2932 | 33172, 33174, 33177, 33189, 33280, 33282, 33288, 33290, 33301, 33312, 33314, 33320, 33322, 33361, 33364, 33369, |
| 2933 | 33381, 33408, 33410, 33416, 33418, 33429, 33440, 33442, 33448, 33450, 33812, 33817, 33857, 33860, 33873, 33877, |
| 2934 | 33882, 33889, 33892, 33897, 33940, 33945, 34049, 34057, 34066, 34069, 34074, 34086, 34089, 34112, 34113, 34117, |
| 2935 | 34120, 34129, 34132, 34133, 34134, 34137, 34138, 34149, 34150, 34152, 34154, 34177, 34180, 34182, 34185, 34192, |
| 2936 | 34194, 34197, 34200, 34214, 34321, 34326, 34329, 34341, 34369, 34372, 34377, 34378, 34384, 34389, 34393, 34394, |
| 2937 | 34401, 34406, 34410, 34437, 34449, 34458, 34468, 34816, 34818, 34824, 34826, 34837, 34848, 34850, 34856, 34858, |
| 2938 | 34881, 34885, 34897, 34900, 34905, 34917, 34921, 34944, 34946, 34952, 34954, 34965, 34976, 34978, 34984, 34986, |
| 2939 | 35077, 35078, 35089, 35092, 35094, 35109, 35137, 35140, 35142, 35145, 35152, 35154, 35157, 35162, 35169, 35172, |
| 2940 | 35205, 35222, 35225, 35237, 35328, 35330, 35336, 35338, 35349, 35360, 35362, 35368, 35370, 35397, 35409, 35412, |
| 2941 | 35414, 35456, 35458, 35464, 35466, 35477, 35488, 35490, 35496, 35498, 36869, 36881, 36886, 36888, 36889, 36901, |
| 2942 | 36929, 36934, 36937, 36949, 36952, 36954, 36969, 36970, 36997, 37009, 37012, 37014, 37017, 37029, 37121, 37124, |
| 2943 | 37126, 37129, 37136, 37141, 37144, 37146, 37153, 37156, 37158, 37161, 37184, 37189, 37200, 37201, 37204, 37205, |
| 2944 | 37206, 37209, 37218, 37221, 37252, 37254, 37266, 37269, 37272, 37281, 37284, 37286, 37289, 37381, 37393, 37396, |
| 2945 | 37401, 37413, 37444, 37446, 37449, 37456, 37458, 37461, 37464, 37478, 37481, 37509, 37524, 37526, 37545, 37889, |
| 2946 | 37892, 37894, 37904, 37909, 37912, 37926, 37952, 37962, 37969, 37972, 37973, 37974, 37976, 37977, 37984, 37985, |
| 2947 | 37986, 37989, 38020, 38022, 38034, 38036, 38037, 38040, 38049, 38057, 38144, 38149, 38152, 38154, 38160, 38161, |
| 2948 | 38164, 38165, 38166, 38169, 38177, 38181, 38185, 38186, 38209, 38212, 38213, 38214, 38217, 38224, 38225, 38226, |
| 2949 | 38228, 38229, 38230, 38232, 38233, 38234, 38241, 38244, 38245, 38246, 38249, 38273, 38277, 38280, 38289, 38290, |
| 2950 | 38292, 38293, 38294, 38297, 38298, 38304, 38306, 38309, 38312, 38314, 38401, 38404, 38416, 38421, 38425, 38432, |
| 2951 | 38438, 38441, 38469, 38472, 38473, 38481, 38482, 38485, 38486, 38489, 38501, 38504, 38530, 38532, 38537, 38538, |
| 2952 | 38546, 38548, 38549, 38564, 38566, 38569, 38917, 38934, 38937, 38949, 38977, 38982, 38992, 38994, 38997, 38998, |
| 2953 | 39002, 39012, 39013, 39045, 39057, 39062, 39065, 39077, 39172, 39174, 39177, 39184, 39186, 39189, 39192, 39194, |
| 2954 | 39200, 39201, 39204, 39206, 39232, 39234, 39237, 39240, 39242, 39249, 39252, 39253, 39254, 39257, 39266, 39269, |
| 2955 | 39270, 39274, 39297, 39300, 39312, 39314, 39317, 39322, 39329, 39334, 39429, 39445, 39461, 39492, 39494, 39497, |
| 2956 | 39504, 39509, 39512, 39521, 39557, 39569, 39572, 39573, 39574, 40960, 40962, 40968, 40970, 40981, 40992, 40994, |
| 2957 | 41000, 41002, 41029, 41041, 41044, 41046, 41049, 41088, 41090, 41096, 41098, 41109, 41120, 41122, 41128, 41130, |
| 2958 | 41221, 41225, 41233, 41236, 41238, 41241, 41242, 41286, 41289, 41297, 41301, 41304, 41306, 41313, 41316, 41349, |
| 2959 | 41360, 41362, 41366, 41369, 41474, 41480, 41482, 41488, 41497, 41506, 41512, 41514, 41541, 41553, 41558, 41561, |
| 2960 | 41573, 41600, 41602, 41608, 41610, 41621, 41632, 41634, 41640, 41642, 42009, 42021, 42049, 42052, 42064, 42068, |
| 2961 | 42069, 42072, 42074, 42081, 42085, 42086, 42088, 42089, 42117, 42246, 42249, 42256, 42258, 42261, 42264, 42278, |
| 2962 | 42281, 42306, 42309, 42321, 42324, 42325, 42326, 42329, 42341, 42346, 42369, 42372, 42373, 42374, 42377, 42386, |
| 2963 | 42389, 42392, 42501, 42513, 42518, 42522, 42529, 42533, 42564, 42566, 42570, 42578, 42581, 42582, 42584, 42592, |
| 2964 | 42594, 42630, 42640, 42645, 42646, 42649, 42657, 42660, 42662, 43008, 43010, 43016, 43018, 43040, 43042, 43048, |
| 2965 | 43050, 43089, 43092, 43094, 43097, 43136, 43138, 43144, 43146, 43157, 43168, 43170, 43176, 43178, 43269, 43284, |
| 2966 | 43289, 43297, 43301, 43329, 43344, 43349, 43354, 43361, 43366, 43369, 43408, 43414, 43520, 43522, 43528, 43530, |
| 2967 | 43552, 43554, 43560, 43562, 43601, 43604, 43606, 43648, 43650, 43656, 43658, 43669, 43680, 43682, 43688, 43690, |
| 2968 | }; |
| 2969 | static const uint16_t kgrid_2bit_1024[1024] = { |
| 2970 | 0, 2, 5, 8, 10, 17, 20, 22, 25, 32, 34, 37, 40, 65, 68, 70, |
| 2971 | 73, 80, 82, 85, 88, 97, 100, 102, 105, 128, 130, 133, 136, 145, 148, 160, |
| 2972 | 165, 170, 257, 260, 262, 265, 272, 274, 277, 280, 289, 292, 320, 322, 325, 328, |
| 2973 | 337, 340, 342, 345, 352, 357, 360, 385, 388, 400, 402, 405, 417, 420, 512, 514, |
| 2974 | 517, 520, 529, 532, 544, 554, 577, 580, 582, 585, 592, 597, 640, 645, 650, 660, |
| 2975 | 674, 1025, 1028, 1030, 1033, 1040, 1042, 1045, 1048, 1057, 1060, 1062, 1065, 1088, 1090, 1093, |
| 2976 | 1096, 1098, 1105, 1108, 1110, 1113, 1120, 1122, 1125, 1153, 1156, 1158, 1161, 1168, 1173, 1176, |
| 2977 | 1185, 1188, 1280, 1282, 1285, 1288, 1290, 1297, 1300, 1302, 1305, 1312, 1317, 1320, 1345, 1348, |
| 2978 | 1350, 1353, 1360, 1362, 1365, 1368, 1377, 1380, 1408, 1410, 1413, 1416, 1425, 1428, 1440, 1537, |
| 2979 | 1540, 1542, 1545, 1552, 1557, 1600, 1605, 1608, 1617, 1620, 1632, 1665, 1668, 1680, 2048, 2050, |
| 2980 | 2053, 2056, 2065, 2068, 2070, 2073, 2080, 2085, 2090, 2113, 2116, 2118, 2121, 2128, 2130, 2133, |
| 2981 | 2136, 2145, 2148, 2176, 2181, 2196, 2218, 2305, 2308, 2320, 2322, 2325, 2328, 2337, 2368, 2373, |
| 2982 | 2376, 2385, 2388, 2400, 2433, 2448, 2560, 2577, 2580, 2594, 2600, 2602, 2640, 2713, 4097, 4100, |
| 2983 | 4102, 4105, 4112, 4114, 4117, 4120, 4129, 4132, 4134, 4160, 4162, 4165, 4168, 4177, 4180, 4182, |
| 2984 | 4185, 4192, 4194, 4197, 4200, 4225, 4228, 4230, 4240, 4245, 4248, 4257, 4260, 4352, 4354, 4357, |
| 2985 | 4360, 4362, 4369, 4372, 4374, 4377, 4384, 4386, 4389, 4392, 4417, 4420, 4422, 4425, 4432, 4434, |
| 2986 | 4437, 4440, 4449, 4452, 4480, 4482, 4485, 4488, 4497, 4500, 4609, 4612, 4617, 4624, 4629, 4641, |
| 2987 | 4644, 4672, 4677, 4689, 4692, 4737, 4740, 4752, 5120, 5122, 5125, 5128, 5137, 5140, 5142, 5145, |
| 2988 | 5152, 5157, 5160, 5185, 5188, 5190, 5193, 5200, 5202, 5205, 5208, 5217, 5220, 5248, 5250, 5253, |
| 2989 | 5256, 5265, 5268, 5280, 5377, 5380, 5382, 5385, 5392, 5394, 5397, 5400, 5409, 5412, 5440, 5442, |
| 2990 | 5445, 5448, 5457, 5460, 5472, 5505, 5508, 5520, 5632, 5637, 5640, 5649, 5652, 5664, 5697, 5700, |
| 2991 | 5712, 5760, 5802, 6145, 6148, 6150, 6153, 6160, 6165, 6168, 6177, 6208, 6210, 6213, 6216, 6225, |
| 2992 | 6228, 6240, 6273, 6276, 6400, 6402, 6405, 6408, 6417, 6420, 6432, 6465, 6468, 6480, 6505, 6562, |
| 2993 | 6660, 6672, 6720, 6742, 8192, 8194, 8197, 8200, 8209, 8212, 8214, 8217, 8224, 8229, 8234, 8257, |
| 2994 | 8260, 8272, 8274, 8277, 8292, 8320, 8330, 8340, 8362, 8449, 8452, 8464, 8466, 8469, 8481, 8512, |
| 2995 | 8514, 8517, 8529, 8532, 8544, 8577, 8580, 8592, 8704, 8714, 8738, 8744, 8746, 8772, 8784, 8840, |
| 2996 | 8842, 8872, 9217, 9220, 9222, 9225, 9232, 9237, 9240, 9249, 9252, 9280, 9282, 9285, 9288, 9297, |
| 2997 | 9300, 9312, 9345, 9348, 9360, 9472, 9477, 9480, 9489, 9492, 9504, 9537, 9540, 9552, 9574, 9600, |
| 2998 | 9729, 9732, 9744, 9792, 9817, 10240, 10245, 10257, 10260, 10305, 10308, 10320, 10378, 10410, 10497, 10500, |
| 2999 | 10512, 10645, 10762, 10786, 10852, 10888, 10890, 16385, 16388, 16390, 16393, 16400, 16402, 16405, 16408, 16410, |
| 3000 | 16417, 16420, 16422, 16448, 16450, 16453, 16456, 16458, 16465, 16468, 16470, 16473, 16480, 16482, 16485, 16513, |
| 3001 | 16516, 16528, 16533, 16536, 16545, 16548, 16640, 16642, 16645, 16648, 16657, 16660, 16662, 16665, 16672, 16674, |
| 3002 | 16677, 16705, 16708, 16710, 16713, 16720, 16722, 16725, 16728, 16737, 16740, 16768, 16770, 16773, 16776, 16785, |
| 3003 | 16788, 16800, 16897, 16900, 16912, 16914, 16917, 16920, 16932, 16960, 16965, 16968, 16977, 16980, 16992, 17025, |
| 3004 | 17028, 17408, 17410, 17413, 17416, 17418, 17425, 17428, 17430, 17433, 17440, 17442, 17445, 17448, 17473, 17476, |
| 3005 | 17478, 17481, 17488, 17490, 17493, 17496, 17505, 17508, 17536, 17538, 17541, 17544, 17553, 17556, 17568, 17665, |
| 3006 | 17668, 17670, 17673, 17680, 17682, 17685, 17688, 17697, 17700, 17728, 17730, 17733, 17736, 17745, 17748, 17760, |
| 3007 | 17770, 17793, 17796, 17808, 17920, 17922, 17925, 17928, 17937, 17940, 17952, 17985, 17988, 18000, 18048, 18085, |
| 3008 | 18433, 18436, 18441, 18448, 18450, 18453, 18456, 18465, 18468, 18496, 18498, 18501, 18504, 18513, 18516, 18528, |
| 3009 | 18564, 18576, 18688, 18690, 18693, 18696, 18705, 18708, 18720, 18753, 18756, 18768, 18816, 18838, 18945, 18948, |
| 3010 | 18960, 19008, 20480, 20482, 20485, 20488, 20497, 20500, 20502, 20505, 20512, 20514, 20517, 20520, 20545, 20548, |
| 3011 | 20550, 20553, 20560, 20562, 20565, 20568, 20577, 20580, 20608, 20610, 20613, 20616, 20625, 20628, 20737, 20740, |
| 3012 | 20742, 20745, 20752, 20754, 20757, 20760, 20769, 20772, 20800, 20802, 20805, 20808, 20817, 20820, 20832, 20865, |
| 3013 | 20868, 20880, 20992, 20997, 21000, 21009, 21012, 21024, 21057, 21060, 21072, 21097, 21120, 21505, 21508, 21510, |
| 3014 | 21513, 21520, 21522, 21525, 21528, 21537, 21540, 21568, 21570, 21573, 21576, 21585, 21588, 21600, 21633, 21636, |
| 3015 | 21648, 21760, 21762, 21765, 21768, 21777, 21780, 21792, 21825, 21828, 21840, 21888, 22017, 22020, 22032, 22054, |
| 3016 | 22080, 22528, 22530, 22533, 22536, 22545, 22548, 22560, 22593, 22596, 22608, 22618, 22656, 22785, 22788, 22800, |
| 3017 | 22848, 23040, 23065, 23173, 23208, 24577, 24580, 24582, 24592, 24594, 24597, 24600, 24609, 24612, 24640, 24645, |
| 3018 | 24648, 24657, 24660, 24672, 24708, 24720, 24832, 24834, 24837, 24840, 24849, 24852, 24864, 24897, 24900, 24912, |
| 3019 | 24960, 24985, 25092, 25104, 25152, 25174, 25249, 25600, 25605, 25608, 25617, 25620, 25632, 25665, 25668, 25680, |
| 3020 | 25728, 25857, 25860, 25872, 25920, 25930, 25960, 26002, 26112, 26260, 26625, 26628, 26640, 26725, 26776, 26880, |
| 3021 | 26922, 27202, 27297, 32768, 32770, 32773, 32776, 32785, 32788, 32793, 32800, 32805, 32833, 32836, 32848, 32850, |
| 3022 | 32853, 32856, 32865, 32896, 32901, 32913, 32916, 33025, 33028, 33033, 33040, 33042, 33045, 33048, 33057, 33060, |
| 3023 | 33088, 33090, 33093, 33096, 33105, 33108, 33153, 33156, 33168, 33193, 33280, 33285, 33290, 33297, 33300, 33345, |
| 3024 | 33348, 33360, 33793, 33796, 33798, 33801, 33808, 33810, 33813, 33816, 33825, 33856, 33858, 33861, 33864, 33873, |
| 3025 | 33876, 33888, 33921, 33924, 33936, 34048, 34050, 34053, 34056, 34065, 34068, 34080, 34113, 34116, 34128, 34176, |
| 3026 | 34186, 34305, 34308, 34320, 34345, 34368, 34816, 34821, 34833, 34836, 34881, 34884, 34896, 34978, 35073, 35076, |
| 3027 | 35136, 35173, 35362, 35416, 35418, 35458, 35490, 36865, 36868, 36873, 36880, 36882, 36885, 36888, 36900, 36928, |
| 3028 | 36930, 36933, 36936, 36945, 36948, 36960, 36993, 36996, 37008, 37120, 37125, 37137, 37140, 37185, 37188, 37200, |
| 3029 | 37210, 37377, 37380, 37392, 37440, 37542, 37888, 37890, 37893, 37896, 37905, 37908, 37920, 37953, 37956, 37968, |
| 3030 | 38016, 38038, 38145, 38148, 38160, 38208, 38296, 38305, 38400, 38470, 38500, 38913, 38916, 38928, 38950, 38976, |
| 3031 | 39081, 39168, 39241, 39250, 39568, 40960, 40965, 40970, 40980, 40994, 41002, 41025, 41028, 41040, 41122, 41130, |
| 3032 | 41280, 41317, 41474, 41482, 41506, 41512, 41514, 41602, 41608, 41610, 41640, 41985, 41988, 42000, 42048, 42121, |
| 3033 | 42148, 42240, 42265, 42577, 43018, 43048, 43170, 43348, 43398, 43528, 43530, 43552, 43554, 43560, 43656, 43690, |
| 3034 | }; |
| 3035 | |
| 3036 | const int kmap_size = 43692; |
| 3037 | //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2; |
| 3038 | const int nwant = type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2; |
| 3039 | const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 : |
| 3040 | type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : |
| 3041 | type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? kgrid_1bit_2048 : kgrid_2bit_1024; |
| 3042 | uint64_t * kgrid_q2xs; |
| 3043 | int * kmap_q2xs; |
| 3044 | uint16_t * kneighbors_q2xs; |
| 3045 | |
| 3046 | //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); |
| 3047 | uint64_t * the_grid = (uint64_t *)malloc(grid_size*sizeof(uint64_t)); |
| 3048 | for (int k = 0; k < grid_size; ++k) { |
| 3049 | int8_t * pos = (int8_t *)(the_grid + k); |
| 3050 | for (int i = 0; i < 8; ++i) { |
| 3051 | int l = (kgrid[k] >> 2*i) & 0x3; |
| 3052 | pos[i] = 2*l + 1; |
| 3053 | } |
| 3054 | } |
| 3055 | kgrid_q2xs = the_grid; |
| 3056 | iq2_data[gindex].grid = the_grid; |
| 3057 | kmap_q2xs = (int *)malloc(kmap_size*sizeof(int)); |
| 3058 | iq2_data[gindex].map = kmap_q2xs; |
| 3059 | for (int i = 0; i < kmap_size; ++i) kmap_q2xs[i] = -1; |
| 3060 | uint64_t aux64; |
| 3061 | uint8_t * aux8 = (uint8_t *)&aux64; |
| 3062 | for (int i = 0; i < grid_size; ++i) { |
| 3063 | aux64 = kgrid_q2xs[i]; |
| 3064 | uint16_t index = 0; |
| 3065 | for (int k=0; k<8; ++k) { |
| 3066 | uint16_t q = (aux8[k] - 1)/2; |
| 3067 | index |= (q << 2*k); |
| 3068 | } |
| 3069 | kmap_q2xs[index] = i; |
| 3070 | } |
| 3071 | // The neighbour search runs in three passes: |
| 3072 | // 1. Parallel: for each i, qsort and count its neighbours into n_per_i, |
| 3073 | // and reduce the totals (num_neighbors, num_not_in_map). |
| 3074 | // 2. Serial: prefix-sum n_per_i into offsets[], so each i has a |
| 3075 | // pre-assigned slice of kneighbors_q2xs to write into. |
| 3076 | // 3. Parallel: redo the qsort and write each i's neighbour list at |
| 3077 | // offsets[i]. |
| 3078 | int * n_per_i = (int *)malloc(kmap_size*sizeof(int)); |
| 3079 | GGML_ASSERT(n_per_i)if (!(n_per_i)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3079, "GGML_ASSERT(%s) failed", "n_per_i"); |
| 3080 | int num_neighbors = 0, num_not_in_map = 0; |
| 3081 | #ifdef GGML_USE_OPENMP |
| 3082 | #pragma omp parallel reduction(+:num_neighbors,num_not_in_map) |
| 3083 | #endif |
| 3084 | { |
| 3085 | int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); |
| 3086 | GGML_ASSERT(dist2)if (!(dist2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3086, "GGML_ASSERT(%s) failed", "dist2"); |
| 3087 | int8_t pos[8]; |
| 3088 | int i; |
| 3089 | #ifdef GGML_USE_OPENMP |
| 3090 | #pragma omp for schedule(dynamic, 64) |
| 3091 | #endif |
| 3092 | for (i = 0; i < kmap_size; ++i) { |
| 3093 | if (kmap_q2xs[i] >= 0) { |
| 3094 | n_per_i[i] = 0; |
| 3095 | continue; |
| 3096 | } |
| 3097 | ++num_not_in_map; |
| 3098 | for (int k = 0; k < 8; ++k) { |
| 3099 | int l = (i >> 2*k) & 0x3; |
| 3100 | pos[k] = 2*l + 1; |
| 3101 | } |
| 3102 | for (int j = 0; j < grid_size; ++j) { |
| 3103 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); |
| 3104 | int d2 = 0; |
| 3105 | for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); |
| 3106 | dist2[2*j+0] = d2; |
| 3107 | dist2[2*j+1] = j; |
| 3108 | } |
| 3109 | qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); |
| 3110 | int n = 0; int d2 = dist2[0]; |
| 3111 | int nhave = 1; |
| 3112 | for (int j = 0; j < grid_size; ++j) { |
| 3113 | if (dist2[2*j] > d2) { |
| 3114 | if (nhave == nwant) break; |
| 3115 | d2 = dist2[2*j]; |
| 3116 | ++nhave; |
| 3117 | } |
| 3118 | ++n; |
| 3119 | } |
| 3120 | n_per_i[i] = n; |
| 3121 | num_neighbors += n; |
| 3122 | } |
| 3123 | free(dist2); |
| 3124 | } |
| 3125 | //printf("%s: %d neighbours in total\n", __func__, num_neighbors); |
| 3126 | kneighbors_q2xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); |
| 3127 | iq2_data[gindex].neighbours = kneighbors_q2xs; |
| 3128 | |
| 3129 | int * offsets = (int *)malloc(kmap_size*sizeof(int)); |
| 3130 | GGML_ASSERT(offsets)if (!(offsets)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3130, "GGML_ASSERT(%s) failed", "offsets"); |
| 3131 | int counter = 0; |
| 3132 | for (int i = 0; i < kmap_size; ++i) { |
| 3133 | if (kmap_q2xs[i] >= 0) { |
| 3134 | offsets[i] = -1; |
| 3135 | continue; |
| 3136 | } |
| 3137 | offsets[i] = counter; |
| 3138 | counter += 1 + n_per_i[i]; |
| 3139 | } |
| 3140 | |
| 3141 | #ifdef GGML_USE_OPENMP |
| 3142 | #pragma omp parallel |
| 3143 | #endif |
| 3144 | { |
| 3145 | int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); |
| 3146 | GGML_ASSERT(dist2)if (!(dist2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3146, "GGML_ASSERT(%s) failed", "dist2"); |
| 3147 | int8_t pos[8]; |
| 3148 | int i; |
| 3149 | #ifdef GGML_USE_OPENMP |
| 3150 | #pragma omp for schedule(dynamic, 64) |
| 3151 | #endif |
| 3152 | for (i = 0; i < kmap_size; ++i) { |
| 3153 | if (kmap_q2xs[i] >= 0) continue; |
| 3154 | for (int k = 0; k < 8; ++k) { |
| 3155 | int l = (i >> 2*k) & 0x3; |
| 3156 | pos[k] = 2*l + 1; |
| 3157 | } |
| 3158 | for (int j = 0; j < grid_size; ++j) { |
| 3159 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + j); |
| 3160 | int d2 = 0; |
| 3161 | for (int k = 0; k < 8; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); |
| 3162 | dist2[2*j+0] = d2; |
| 3163 | dist2[2*j+1] = j; |
| 3164 | } |
| 3165 | qsort(dist2, grid_size, 2*sizeof(int), iq2_compare_func); |
| 3166 | int local_counter = offsets[i]; |
| 3167 | kmap_q2xs[i] = -(local_counter + 1); |
| 3168 | int d2 = dist2[0]; |
| 3169 | uint16_t * start = &kneighbors_q2xs[local_counter++]; |
| 3170 | int n = 0, nhave = 1; |
| 3171 | for (int j = 0; j < grid_size; ++j) { |
| 3172 | if (dist2[2*j] > d2) { |
| 3173 | if (nhave == nwant) break; |
| 3174 | d2 = dist2[2*j]; |
| 3175 | ++nhave; |
| 3176 | } |
| 3177 | kneighbors_q2xs[local_counter++] = dist2[2*j+1]; |
| 3178 | ++n; |
| 3179 | } |
| 3180 | *start = n; |
| 3181 | } |
| 3182 | free(dist2); |
| 3183 | } |
| 3184 | free(offsets); |
| 3185 | free(n_per_i); |
| 3186 | } |
| 3187 | |
| 3188 | void iq2xs_free_impl(enum ggml_type type) { |
| 3189 | GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)if (!(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3189, "GGML_ASSERT(%s) failed", "type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S" ); |
| 3190 | const int gindex = iq2_data_index(type); |
| 3191 | if (iq2_data[gindex].grid) { |
| 3192 | free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL((void*)0); |
| 3193 | free(iq2_data[gindex].map); iq2_data[gindex].map = NULL((void*)0); |
| 3194 | free(iq2_data[gindex].neighbours); iq2_data[gindex].neighbours = NULL((void*)0); |
| 3195 | } |
| 3196 | } |
| 3197 | |
| 3198 | static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICTrestrict neighbours, const uint64_t * GGML_RESTRICTrestrict grid, |
| 3199 | const float * GGML_RESTRICTrestrict xval, const float * GGML_RESTRICTrestrict weight, float scale, int8_t * GGML_RESTRICTrestrict L) { |
| 3200 | int num_neighbors = neighbours[0]; |
| 3201 | GGML_ASSERT(num_neighbors > 0)if (!(num_neighbors > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3201, "GGML_ASSERT(%s) failed", "num_neighbors > 0"); |
| 3202 | float best_d2 = FLT_MAX3.40282347e+38F; |
| 3203 | int grid_index = -1; |
| 3204 | for (int j = 1; j <= num_neighbors; ++j) { |
| 3205 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 3206 | float d2 = 0; |
| 3207 | for (int i = 0; i < 8; ++i) { |
| 3208 | float q = pg[i]; |
| 3209 | float diff = scale*q - xval[i]; |
| 3210 | d2 += weight[i]*diff*diff; |
| 3211 | } |
| 3212 | if (d2 < best_d2) { |
| 3213 | best_d2 = d2; grid_index = neighbours[j]; |
| 3214 | } |
| 3215 | } |
| 3216 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3216, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 3217 | const int8_t * pg = (const int8_t *)(grid + grid_index); |
| 3218 | for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; |
| 3219 | return grid_index; |
| 3220 | } |
| 3221 | |
| 3222 | static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, const float * GGML_RESTRICTrestrict quant_weights) { |
| 3223 | |
| 3224 | const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS); |
| 3225 | |
| 3226 | const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; |
| 3227 | const int * kmap_q2xs = iq2_data[gindex].map; |
| 3228 | const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; |
| 3229 | |
| 3230 | GGML_ASSERT(quant_weights && "missing quantization weights")if (!(quant_weights && "missing quantization weights" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3230, "GGML_ASSERT(%s) failed", "quant_weights && \"missing quantization weights\"" ); |
| 3231 | GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3231, "GGML_ASSERT(%s) failed", "kgrid_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3232 | GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3232, "GGML_ASSERT(%s) failed", "kmap_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3233 | GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3233, "GGML_ASSERT(%s) failed", "kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3234 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3234, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 3235 | |
| 3236 | const int kMaxQ = 3; |
| 3237 | |
| 3238 | const int64_t nbl = n/QK_K256; |
| 3239 | |
| 3240 | block_iq2_xxs * y = vy; |
| 3241 | |
| 3242 | float scales[QK_K256/32]; |
| 3243 | float weight[32]; |
| 3244 | float xval[32]; |
| 3245 | int8_t L[32]; |
| 3246 | int8_t Laux[32]; |
| 3247 | float waux[32]; |
| 3248 | uint8_t block_signs[4]; |
| 3249 | uint32_t q2[2*(QK_K256/32)]; |
| 3250 | |
| 3251 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 3252 | |
| 3253 | y[ibl].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 3254 | memset(q2, 0, QK_K256/4); |
| 3255 | |
| 3256 | float max_scale = 0; |
| 3257 | |
| 3258 | const float * xbl = x + QK_K256*ibl; |
| 3259 | float sumx2 = 0; |
| 3260 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 3261 | float sigma2 = sumx2/QK_K256; |
| 3262 | |
| 3263 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 3264 | const float * xb = xbl + 32*ib; |
| 3265 | const float * qw = quant_weights + QK_K256*ibl + 32*ib; |
| 3266 | for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 3267 | for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]); |
| 3268 | for (int k = 0; k < 4; ++k) { |
| 3269 | int nflip = 0; |
| 3270 | uint8_t s = 0; |
| 3271 | for (int i = 0; i < 8; ++i) { |
| 3272 | if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; |
| 3273 | else { |
| 3274 | xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i); |
| 3275 | } |
| 3276 | } |
| 3277 | if (nflip%2) { |
| 3278 | int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin]; |
| 3279 | for (int i = 1; i < 8; ++i) { |
| 3280 | float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i]; |
| 3281 | if (ax < min) { |
| 3282 | min = ax; imin = i; |
| 3283 | } |
| 3284 | } |
| 3285 | xval[8*k+imin] = -xval[8*k+imin]; |
| 3286 | s ^= (1 << imin); |
| 3287 | } |
| 3288 | block_signs[k] = s & 127; |
| 3289 | } |
| 3290 | float max = xval[0]; |
| 3291 | for (int i = 1; i < 32; ++i) max = MAX(max, xval[i])((max) > (xval[i]) ? (max) : (xval[i])); |
| 3292 | if (max < GROUP_MAX_EPS1e-15f) { |
| 3293 | scales[ib] = 0; |
| 3294 | memset(L, 0, 32); |
| 3295 | continue; |
| 3296 | } |
| 3297 | float scale = make_qp_quants(32, kMaxQ+1, xval, (uint8_t*)L, weight); |
| 3298 | float eff_max = scale*kMaxQ; |
| 3299 | if (eff_max <= 0) { |
| 3300 | scales[ib] = 0; |
| 3301 | memset(L, 0, 32); |
| 3302 | continue; |
| 3303 | } |
| 3304 | float best = 0; |
| 3305 | for (int is = -6; is <= 6; ++is) { |
| 3306 | float id = (2*kMaxQ-1+is*0.1f)/eff_max; |
| 3307 | float this_scale = 1/id; |
| 3308 | for (int k = 0; k < 4; ++k) { |
| 3309 | for (int i = 0; i < 8; ++i) { |
| 3310 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 3311 | Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 3312 | } |
| 3313 | uint16_t u = 0; |
| 3314 | for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); |
| 3315 | int grid_index = kmap_q2xs[u]; |
| 3316 | if (grid_index < 0) { |
| 3317 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 3318 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); |
| 3319 | } |
| 3320 | } |
| 3321 | float sumqx = 0, sumq2 = 0; |
| 3322 | for (int i = 0; i < 32; ++i) { |
| 3323 | float w = weight[i]; |
| 3324 | float q = 2*Laux[i] + 1; |
| 3325 | sumqx += w*xval[i]*q; |
| 3326 | sumq2 += w*q*q; |
| 3327 | } |
| 3328 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 3329 | scale = sumqx/sumq2; best = scale*sumqx; |
| 3330 | memcpy(L, Laux, 32); |
| 3331 | } |
| 3332 | } |
| 3333 | if (scale > 0) { |
| 3334 | float id = 1/scale; |
| 3335 | for (int k = 0; k < 4; ++k) { |
| 3336 | uint16_t u = 0; |
| 3337 | for (int i = 0; i < 8; ++i) { |
| 3338 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 3339 | l = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 3340 | u |= (l << 2*i); |
| 3341 | } |
| 3342 | int grid_index = kmap_q2xs[u]; |
| 3343 | if (grid_index < 0) { |
| 3344 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 3345 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); |
| 3346 | } |
| 3347 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index); |
| 3348 | for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2; |
| 3349 | } |
| 3350 | float sumqx = 0, sumq2 = 0; |
| 3351 | for (int i = 0; i < 32; ++i) { |
| 3352 | float w = weight[i]; |
| 3353 | float q = 2*L[i] + 1; |
| 3354 | sumqx += w*xval[i]*q; |
| 3355 | sumq2 += w*q*q; |
| 3356 | } |
| 3357 | if (sumq2 > 0) scale = sumqx/sumq2; |
| 3358 | } |
| 3359 | if (scale < 0) { |
| 3360 | // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale) |
| 3361 | // and correspondingly flip quant signs. |
| 3362 | scale = -scale; |
| 3363 | for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127; |
| 3364 | } |
| 3365 | for (int k = 0; k < 4; ++k) { |
| 3366 | uint16_t u = 0; |
| 3367 | for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); |
| 3368 | int grid_index = kmap_q2xs[u]; |
| 3369 | if (grid_index < 0) { |
| 3370 | printf("Oops: found point %u not on grid:", u); |
| 3371 | for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); |
| 3372 | printf("\n"); |
| 3373 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3373, "fatal error"); |
| 3374 | } |
| 3375 | q2[2*ib+0] |= ((uint32_t) grid_index << 8*k); |
| 3376 | q2[2*ib+1] |= (block_signs[k] << 7*k); |
| 3377 | } |
| 3378 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3378, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 3379 | scales[ib] = scale; |
| 3380 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 3381 | } |
| 3382 | |
| 3383 | if (!max_scale) { |
| 3384 | memset(y[ibl].qs, 0, QK_K256/4); |
| 3385 | continue; |
| 3386 | } |
| 3387 | |
| 3388 | float d = max_scale/31; |
| 3389 | y[ibl].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 3390 | float id = 1/d; |
| 3391 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 3392 | int l = nearest_int(0.5f*(id*scales[ib]-1)); |
| 3393 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 3394 | q2[2*ib+1] |= ((uint32_t)l << 28); |
| 3395 | } |
| 3396 | memcpy(y[ibl].qs, q2, QK_K256/4); |
| 3397 | } |
| 3398 | } |
| 3399 | |
| 3400 | static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, const float * GGML_RESTRICTrestrict quant_weights) { |
| 3401 | |
| 3402 | const int gindex = iq2_data_index(GGML_TYPE_IQ2_XS); |
| 3403 | |
| 3404 | const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; |
| 3405 | const int * kmap_q2xs = iq2_data[gindex].map; |
| 3406 | const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; |
| 3407 | |
| 3408 | GGML_ASSERT(quant_weights && "missing quantization weights")if (!(quant_weights && "missing quantization weights" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3408, "GGML_ASSERT(%s) failed", "quant_weights && \"missing quantization weights\"" ); |
| 3409 | GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3409, "GGML_ASSERT(%s) failed", "kmap_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3410 | GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3410, "GGML_ASSERT(%s) failed", "kgrid_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3411 | GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3411, "GGML_ASSERT(%s) failed", "kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3412 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3412, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 3413 | |
| 3414 | const int kMaxQ = 3; |
| 3415 | |
| 3416 | const int64_t nbl = n/QK_K256; |
| 3417 | |
| 3418 | block_iq2_xs * y = vy; |
| 3419 | |
| 3420 | float scales[QK_K256/16]; |
| 3421 | float weight[16]; |
| 3422 | float xval[16]; |
| 3423 | int8_t L[16]; |
| 3424 | int8_t Laux[16]; |
| 3425 | float waux[16]; |
| 3426 | bool_Bool is_on_grid[2]; |
| 3427 | bool_Bool is_on_grid_aux[2]; |
| 3428 | uint8_t block_signs[2]; |
| 3429 | uint16_t q2[2*(QK_K256/16)]; |
| 3430 | |
| 3431 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 3432 | |
| 3433 | y[ibl].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 3434 | memset(q2, 0, QK_K256/4); |
| 3435 | memset(y[ibl].scales, 0, QK_K256/32); |
| 3436 | |
| 3437 | float max_scale = 0; |
| 3438 | |
| 3439 | const float * xbl = x + QK_K256*ibl; |
| 3440 | float sumx2 = 0; |
| 3441 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 3442 | float sigma2 = sumx2/QK_K256; |
| 3443 | |
| 3444 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 3445 | const float * xb = xbl + 16*ib; |
| 3446 | const float * qw = quant_weights + QK_K256*ibl + 16*ib; |
| 3447 | for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 3448 | for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]); |
| 3449 | for (int k = 0; k < 2; ++k) { |
| 3450 | int nflip = 0; |
| 3451 | uint8_t s = 0; |
| 3452 | for (int i = 0; i < 8; ++i) { |
| 3453 | if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; |
| 3454 | else { |
| 3455 | xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i); |
| 3456 | } |
| 3457 | } |
| 3458 | if (nflip%2) { |
| 3459 | int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin]; |
| 3460 | for (int i = 1; i < 8; ++i) { |
| 3461 | float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i]; |
| 3462 | if (ax < min) { |
| 3463 | min = ax; imin = i; |
| 3464 | } |
| 3465 | } |
| 3466 | xval[8*k+imin] = -xval[8*k+imin]; |
| 3467 | s ^= (1 << imin); |
| 3468 | } |
| 3469 | block_signs[k] = s & 127; |
| 3470 | } |
| 3471 | float max = xval[0]; |
| 3472 | for (int i = 1; i < 16; ++i) max = MAX(max, xval[i])((max) > (xval[i]) ? (max) : (xval[i])); |
| 3473 | memset(L, 0, 16); |
| 3474 | if (max < GROUP_MAX_EPS1e-15f) { |
| 3475 | scales[ib] = 0; |
| 3476 | continue; |
| 3477 | } |
| 3478 | float best = 0; |
| 3479 | float scale = max/(2*kMaxQ-1); |
| 3480 | is_on_grid[0] = is_on_grid[1] = true1; |
| 3481 | for (int is = -9; is <= 9; ++is) { |
| 3482 | float id = (2*kMaxQ-1+is*0.1f)/max; |
| 3483 | float this_scale = 1/id; |
| 3484 | for (int k = 0; k < 2; ++k) { |
| 3485 | for (int i = 0; i < 8; ++i) { |
| 3486 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 3487 | Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 3488 | } |
| 3489 | uint16_t u = 0; |
| 3490 | for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); |
| 3491 | int grid_index = kmap_q2xs[u]; |
| 3492 | is_on_grid_aux[k] = true1; |
| 3493 | if (grid_index < 0) { |
| 3494 | is_on_grid_aux[k] = false0; |
| 3495 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 3496 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); |
| 3497 | } |
| 3498 | } |
| 3499 | float sumqx = 0, sumq2 = 0; |
| 3500 | for (int i = 0; i < 16; ++i) { |
| 3501 | float w = weight[i]; |
| 3502 | float q = 2*Laux[i] + 1; |
| 3503 | sumqx += w*xval[i]*q; |
| 3504 | sumq2 += w*q*q; |
| 3505 | } |
| 3506 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 3507 | scale = sumqx/sumq2; best = scale*sumqx; |
| 3508 | for (int i = 0; i < 16; ++i) L[i] = Laux[i]; |
| 3509 | for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k]; |
| 3510 | } |
| 3511 | } |
| 3512 | int n_not_ongrid = 0; |
| 3513 | for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid; |
| 3514 | if (n_not_ongrid > 0 && scale > 0) { |
| 3515 | float id = 1/scale; |
| 3516 | for (int k = 0; k < 2; ++k) { |
| 3517 | if (is_on_grid[k]) continue; |
| 3518 | uint16_t u = 0; |
| 3519 | for (int i = 0; i < 8; ++i) { |
| 3520 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 3521 | l = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 3522 | u |= (l << 2*i); |
| 3523 | L[8*k + i] = l; |
| 3524 | } |
| 3525 | int grid_index = kmap_q2xs[u]; |
| 3526 | if (grid_index < 0) { |
| 3527 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 3528 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); |
Value stored to 'grid_index' is never read | |
| 3529 | } |
| 3530 | } |
| 3531 | float sumqx = 0, sumq2 = 0; |
| 3532 | for (int i = 0; i < 16; ++i) { |
| 3533 | float w = weight[i]; |
| 3534 | float q = 2*L[i] + 1; |
| 3535 | sumqx += w*xval[i]*q; |
| 3536 | sumq2 += w*q*q; |
| 3537 | } |
| 3538 | if (sumq2 > 0) scale = sumqx/sumq2; |
| 3539 | } |
| 3540 | if (scale < 0) { |
| 3541 | scale = -scale; |
| 3542 | for (int k = 0; k < 2; ++k) block_signs[k] = (~block_signs[k]) & 127; |
| 3543 | } |
| 3544 | for (int k = 0; k < 2; ++k) { |
| 3545 | uint16_t u = 0; |
| 3546 | for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); |
| 3547 | int grid_index = kmap_q2xs[u]; |
| 3548 | if (grid_index < 0) { |
| 3549 | printf("Oops: found point %u not on grid:", u); |
| 3550 | for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); |
| 3551 | printf("\n"); |
| 3552 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3552, "fatal error"); |
| 3553 | } |
| 3554 | q2[2*ib+k] = grid_index | (block_signs[k] << 9); |
| 3555 | } |
| 3556 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3556, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 3557 | scales[ib] = scale; |
| 3558 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 3559 | } |
| 3560 | |
| 3561 | if (!max_scale) { |
| 3562 | memset(y[ibl].qs, 0, QK_K256/4); |
| 3563 | continue; |
| 3564 | } |
| 3565 | |
| 3566 | float d = max_scale/31; |
| 3567 | y[ibl].d = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 3568 | float id = 1/d; |
| 3569 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 3570 | int l = nearest_int(0.5f*(id*scales[ib]-1)); |
| 3571 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 3572 | if (ib%2 == 0) y[ibl].scales[ib/2] = l; |
| 3573 | else y[ibl].scales[ib/2] |= (l << 4); |
| 3574 | } |
| 3575 | memcpy(y[ibl].qs, q2, QK_K256/4); |
| 3576 | |
| 3577 | } |
| 3578 | } |
| 3579 | |
| 3580 | size_t quantize_iq2_xxs(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 3581 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3581, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 3582 | int64_t nblock = n_per_row/QK_K256; |
| 3583 | char * qrow = (char *)dst; |
| 3584 | for (int64_t row = 0; row < nrow; ++row) { |
| 3585 | quantize_row_iq2_xxs_impl(src, qrow, n_per_row, quant_weights); |
| 3586 | src += n_per_row; |
| 3587 | qrow += nblock*sizeof(block_iq2_xxs); |
| 3588 | } |
| 3589 | return nrow * nblock * sizeof(block_iq2_xxs); |
| 3590 | } |
| 3591 | |
| 3592 | size_t quantize_iq2_xs(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 3593 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3593, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 3594 | int64_t nblock = n_per_row/QK_K256; |
| 3595 | char * qrow = (char *)dst; |
| 3596 | for (int64_t row = 0; row < nrow; ++row) { |
| 3597 | quantize_row_iq2_xs_impl(src, qrow, n_per_row, quant_weights); |
| 3598 | src += n_per_row; |
| 3599 | qrow += nblock*sizeof(block_iq2_xs); |
| 3600 | } |
| 3601 | return nrow * nblock * sizeof(block_iq2_xs); |
| 3602 | } |
| 3603 | |
| 3604 | // |
| 3605 | // ============================================= 3-bit using D4 lattice |
| 3606 | // |
| 3607 | |
| 3608 | typedef struct { |
| 3609 | uint32_t * grid; |
| 3610 | int * map; |
| 3611 | uint16_t * neighbours; |
| 3612 | } iq3_entry_t; |
| 3613 | |
| 3614 | static iq3_entry_t iq3_data[2] = { |
| 3615 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 3616 | {NULL((void*)0), NULL((void*)0), NULL((void*)0)}, |
| 3617 | }; |
| 3618 | |
| 3619 | static inline int iq3_data_index(int grid_size) { |
| 3620 | (void)grid_size; |
| 3621 | GGML_ASSERT(grid_size == 256 || grid_size == 512)if (!(grid_size == 256 || grid_size == 512)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3621, "GGML_ASSERT(%s) failed", "grid_size == 256 || grid_size == 512" ); |
| 3622 | return grid_size == 256 ? 0 : 1; |
| 3623 | } |
| 3624 | |
| 3625 | static int iq3_compare_func(const void * left, const void * right) { |
| 3626 | const int * l = (const int *)left; |
| 3627 | const int * r = (const int *)right; |
| 3628 | return l[0] < r[0] ? -1 : l[0] > r[0] ? 1 : l[1] < r[1] ? -1 : l[1] > r[1] ? 1 : 0; |
| 3629 | } |
| 3630 | |
| 3631 | void iq3xs_init_impl(int grid_size) { |
| 3632 | const int gindex = iq3_data_index(grid_size); |
| 3633 | if (iq3_data[gindex].grid) { |
| 3634 | return; |
| 3635 | } |
| 3636 | static const uint16_t kgrid_256[256] = { |
| 3637 | 0, 2, 4, 9, 11, 15, 16, 18, 25, 34, 59, 61, 65, 67, 72, 74, |
| 3638 | 81, 85, 88, 90, 97, 108, 120, 128, 130, 132, 137, 144, 146, 153, 155, 159, |
| 3639 | 169, 175, 189, 193, 199, 200, 202, 213, 248, 267, 287, 292, 303, 315, 317, 321, |
| 3640 | 327, 346, 362, 413, 436, 456, 460, 462, 483, 497, 513, 515, 520, 522, 529, 531, |
| 3641 | 536, 538, 540, 551, 552, 576, 578, 585, 592, 594, 641, 643, 648, 650, 657, 664, |
| 3642 | 698, 704, 706, 720, 729, 742, 758, 769, 773, 808, 848, 852, 870, 889, 901, 978, |
| 3643 | 992, 1024, 1026, 1033, 1035, 1040, 1042, 1046, 1049, 1058, 1089, 1091, 1093, 1096, 1098, 1105, |
| 3644 | 1112, 1139, 1143, 1144, 1152, 1154, 1161, 1167, 1168, 1170, 1183, 1184, 1197, 1217, 1224, 1228, |
| 3645 | 1272, 1276, 1309, 1323, 1347, 1367, 1377, 1404, 1473, 1475, 1486, 1509, 1537, 1544, 1546, 1553, |
| 3646 | 1555, 1576, 1589, 1594, 1600, 1602, 1616, 1625, 1636, 1638, 1665, 1667, 1672, 1685, 1706, 1722, |
| 3647 | 1737, 1755, 1816, 1831, 1850, 1856, 1862, 1874, 1901, 1932, 1950, 1971, 2011, 2032, 2052, 2063, |
| 3648 | 2077, 2079, 2091, 2095, 2172, 2192, 2207, 2208, 2224, 2230, 2247, 2277, 2308, 2345, 2356, 2389, |
| 3649 | 2403, 2424, 2501, 2504, 2506, 2520, 2570, 2593, 2616, 2624, 2630, 2646, 2669, 2700, 2714, 2746, |
| 3650 | 2754, 2795, 2824, 2835, 2839, 2874, 2882, 2905, 2984, 3028, 3042, 3092, 3108, 3110, 3124, 3153, |
| 3651 | 3185, 3215, 3252, 3288, 3294, 3364, 3397, 3434, 3483, 3523, 3537, 3587, 3589, 3591, 3592, 3610, |
| 3652 | 3626, 3670, 3680, 3722, 3749, 3754, 3776, 3789, 3803, 3824, 3857, 3873, 3904, 3906, 3924, 3992, |
| 3653 | }; |
| 3654 | static const uint16_t kgrid_512[512] = { |
| 3655 | 0, 1, 2, 5, 7, 8, 9, 10, 12, 14, 16, 17, 21, 27, 32, 34, |
| 3656 | 37, 39, 41, 43, 48, 50, 57, 60, 63, 64, 65, 66, 68, 72, 73, 77, |
| 3657 | 80, 83, 87, 89, 93, 100, 113, 117, 122, 128, 129, 133, 135, 136, 139, 142, |
| 3658 | 145, 149, 152, 156, 162, 165, 167, 169, 171, 184, 187, 195, 201, 205, 208, 210, |
| 3659 | 217, 219, 222, 228, 232, 234, 247, 249, 253, 256, 267, 271, 273, 276, 282, 288, |
| 3660 | 291, 297, 312, 322, 324, 336, 338, 342, 347, 353, 357, 359, 374, 379, 390, 393, |
| 3661 | 395, 409, 426, 441, 448, 450, 452, 464, 466, 470, 475, 488, 492, 512, 513, 514, |
| 3662 | 516, 520, 521, 523, 525, 527, 528, 530, 537, 540, 542, 556, 558, 561, 570, 576, |
| 3663 | 577, 579, 582, 584, 588, 593, 600, 603, 609, 616, 618, 632, 638, 640, 650, 653, |
| 3664 | 655, 656, 660, 666, 672, 675, 685, 688, 698, 705, 708, 711, 712, 715, 721, 727, |
| 3665 | 728, 732, 737, 754, 760, 771, 773, 778, 780, 793, 795, 802, 806, 808, 812, 833, |
| 3666 | 840, 843, 849, 856, 858, 873, 912, 916, 919, 932, 934, 961, 963, 968, 970, 977, |
| 3667 | 989, 993, 1010, 1016, 1024, 1025, 1027, 1029, 1031, 1032, 1034, 1036, 1038, 1041, 1043, 1047, |
| 3668 | 1048, 1050, 1057, 1059, 1061, 1064, 1066, 1079, 1080, 1083, 1085, 1088, 1090, 1096, 1099, 1103, |
| 3669 | 1106, 1109, 1113, 1116, 1122, 1129, 1153, 1156, 1159, 1169, 1171, 1176, 1183, 1185, 1195, 1199, |
| 3670 | 1209, 1212, 1216, 1218, 1221, 1225, 1234, 1236, 1241, 1243, 1250, 1256, 1270, 1281, 1287, 1296, |
| 3671 | 1299, 1306, 1309, 1313, 1338, 1341, 1348, 1353, 1362, 1375, 1376, 1387, 1400, 1408, 1410, 1415, |
| 3672 | 1425, 1453, 1457, 1477, 1481, 1494, 1496, 1507, 1512, 1538, 1545, 1547, 1549, 1551, 1554, 1561, |
| 3673 | 1563, 1565, 1570, 1572, 1575, 1577, 1587, 1593, 1601, 1603, 1605, 1612, 1617, 1619, 1632, 1648, |
| 3674 | 1658, 1662, 1664, 1674, 1680, 1690, 1692, 1704, 1729, 1736, 1740, 1745, 1747, 1751, 1752, 1761, |
| 3675 | 1763, 1767, 1773, 1787, 1795, 1801, 1806, 1810, 1817, 1834, 1840, 1844, 1857, 1864, 1866, 1877, |
| 3676 | 1882, 1892, 1902, 1915, 1934, 1953, 1985, 1987, 2000, 2002, 2013, 2048, 2052, 2058, 2064, 2068, |
| 3677 | 2071, 2074, 2081, 2088, 2104, 2114, 2119, 2121, 2123, 2130, 2136, 2141, 2147, 2153, 2157, 2177, |
| 3678 | 2179, 2184, 2189, 2193, 2203, 2208, 2223, 2226, 2232, 2244, 2249, 2251, 2256, 2258, 2265, 2269, |
| 3679 | 2304, 2306, 2324, 2335, 2336, 2361, 2373, 2375, 2385, 2418, 2443, 2460, 2480, 2504, 2509, 2520, |
| 3680 | 2531, 2537, 2562, 2568, 2572, 2578, 2592, 2596, 2599, 2602, 2614, 2620, 2625, 2627, 2629, 2634, |
| 3681 | 2641, 2650, 2682, 2688, 2697, 2707, 2712, 2718, 2731, 2754, 2759, 2760, 2775, 2788, 2793, 2805, |
| 3682 | 2811, 2817, 2820, 2832, 2842, 2854, 2890, 2902, 2921, 2923, 2978, 3010, 3012, 3026, 3081, 3083, |
| 3683 | 3085, 3097, 3099, 3120, 3136, 3152, 3159, 3188, 3210, 3228, 3234, 3245, 3250, 3256, 3264, 3276, |
| 3684 | 3281, 3296, 3349, 3363, 3378, 3392, 3395, 3420, 3440, 3461, 3488, 3529, 3531, 3584, 3588, 3591, |
| 3685 | 3600, 3602, 3614, 3616, 3628, 3634, 3650, 3657, 3668, 3683, 3685, 3713, 3716, 3720, 3726, 3729, |
| 3686 | 3736, 3753, 3778, 3802, 3805, 3819, 3841, 3845, 3851, 3856, 3880, 3922, 3938, 3970, 3993, 4032, |
| 3687 | }; |
| 3688 | |
| 3689 | const int kmap_size = 4096; |
| 3690 | const int nwant = grid_size == 256 ? 2 : 3; |
| 3691 | const uint16_t * kgrid = grid_size == 256 ? kgrid_256 : kgrid_512; |
| 3692 | uint32_t * kgrid_q3xs; |
| 3693 | int * kmap_q3xs; |
| 3694 | uint16_t * kneighbors_q3xs; |
| 3695 | |
| 3696 | //printf("================================================================= %s(grid_size = %d)\n", __func__, grid_size); |
| 3697 | uint32_t * the_grid = (uint32_t *)malloc(grid_size*sizeof(uint32_t)); |
| 3698 | for (int k = 0; k < grid_size; ++k) { |
| 3699 | int8_t * pos = (int8_t *)(the_grid + k); |
| 3700 | for (int i = 0; i < 4; ++i) { |
| 3701 | int l = (kgrid[k] >> 3*i) & 0x7; |
| 3702 | pos[i] = 2*l + 1; |
| 3703 | } |
| 3704 | } |
| 3705 | kgrid_q3xs = the_grid; |
| 3706 | iq3_data[gindex].grid = the_grid; |
| 3707 | kmap_q3xs = (int *)malloc(kmap_size*sizeof(int)); |
| 3708 | iq3_data[gindex].map = kmap_q3xs; |
| 3709 | for (int i = 0; i < kmap_size; ++i) kmap_q3xs[i] = -1; |
| 3710 | uint32_t aux32; |
| 3711 | uint8_t * aux8 = (uint8_t *)&aux32; |
| 3712 | for (int i = 0; i < grid_size; ++i) { |
| 3713 | aux32 = kgrid_q3xs[i]; |
| 3714 | uint16_t index = 0; |
| 3715 | for (int k=0; k<4; ++k) { |
| 3716 | uint16_t q = (aux8[k] - 1)/2; |
| 3717 | index |= (q << 3*k); |
| 3718 | } |
| 3719 | kmap_q3xs[index] = i; |
| 3720 | } |
| 3721 | // See explanation of parallelism in iq2xs_init_impl |
| 3722 | int * n_per_i = (int *)malloc(kmap_size*sizeof(int)); |
| 3723 | GGML_ASSERT(n_per_i)if (!(n_per_i)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3723, "GGML_ASSERT(%s) failed", "n_per_i"); |
| 3724 | int num_neighbors = 0, num_not_in_map = 0; |
| 3725 | #ifdef GGML_USE_OPENMP |
| 3726 | #pragma omp parallel reduction(+:num_neighbors,num_not_in_map) |
| 3727 | #endif |
| 3728 | { |
| 3729 | int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); |
| 3730 | GGML_ASSERT(dist2)if (!(dist2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3730, "GGML_ASSERT(%s) failed", "dist2"); |
| 3731 | int8_t pos[4]; |
| 3732 | int i; |
| 3733 | #ifdef GGML_USE_OPENMP |
| 3734 | #pragma omp for schedule(dynamic, 64) |
| 3735 | #endif |
| 3736 | for (i = 0; i < kmap_size; ++i) { |
| 3737 | if (kmap_q3xs[i] >= 0) { |
| 3738 | n_per_i[i] = 0; |
| 3739 | continue; |
| 3740 | } |
| 3741 | ++num_not_in_map; |
| 3742 | for (int k = 0; k < 4; ++k) { |
| 3743 | int l = (i >> 3*k) & 0x7; |
| 3744 | pos[k] = 2*l + 1; |
| 3745 | } |
| 3746 | for (int j = 0; j < grid_size; ++j) { |
| 3747 | const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); |
| 3748 | int d2 = 0; |
| 3749 | for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); |
| 3750 | dist2[2*j+0] = d2; |
| 3751 | dist2[2*j+1] = j; |
| 3752 | } |
| 3753 | qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); |
| 3754 | int n = 0; int d2 = dist2[0]; |
| 3755 | int nhave = 1; |
| 3756 | for (int j = 0; j < grid_size; ++j) { |
| 3757 | if (dist2[2*j] > d2) { |
| 3758 | if (nhave == nwant) break; |
| 3759 | d2 = dist2[2*j]; |
| 3760 | ++nhave; |
| 3761 | } |
| 3762 | ++n; |
| 3763 | } |
| 3764 | n_per_i[i] = n; |
| 3765 | num_neighbors += n; |
| 3766 | } |
| 3767 | free(dist2); |
| 3768 | } |
| 3769 | //printf("%s: %d neighbours in total\n", __func__, num_neighbors); |
| 3770 | kneighbors_q3xs = (uint16_t *)malloc((num_neighbors + num_not_in_map)*sizeof(uint16_t)); |
| 3771 | iq3_data[gindex].neighbours = kneighbors_q3xs; |
| 3772 | |
| 3773 | int * offsets = (int *)malloc(kmap_size*sizeof(int)); |
| 3774 | GGML_ASSERT(offsets)if (!(offsets)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3774, "GGML_ASSERT(%s) failed", "offsets"); |
| 3775 | int counter = 0; |
| 3776 | for (int i = 0; i < kmap_size; ++i) { |
| 3777 | if (kmap_q3xs[i] >= 0) { |
| 3778 | offsets[i] = -1; |
| 3779 | continue; |
| 3780 | } |
| 3781 | offsets[i] = counter; |
| 3782 | counter += 1 + n_per_i[i]; |
| 3783 | } |
| 3784 | |
| 3785 | #ifdef GGML_USE_OPENMP |
| 3786 | #pragma omp parallel |
| 3787 | #endif |
| 3788 | { |
| 3789 | int * dist2 = (int *)malloc(2*grid_size*sizeof(int)); |
| 3790 | GGML_ASSERT(dist2)if (!(dist2)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3790, "GGML_ASSERT(%s) failed", "dist2"); |
| 3791 | int8_t pos[4]; |
| 3792 | int i; |
| 3793 | #ifdef GGML_USE_OPENMP |
| 3794 | #pragma omp for schedule(dynamic, 64) |
| 3795 | #endif |
| 3796 | for (i = 0; i < kmap_size; ++i) { |
| 3797 | if (kmap_q3xs[i] >= 0) continue; |
| 3798 | for (int k = 0; k < 4; ++k) { |
| 3799 | int l = (i >> 3*k) & 0x7; |
| 3800 | pos[k] = 2*l + 1; |
| 3801 | } |
| 3802 | for (int j = 0; j < grid_size; ++j) { |
| 3803 | const int8_t * pg = (const int8_t *)(kgrid_q3xs + j); |
| 3804 | int d2 = 0; |
| 3805 | for (int k = 0; k < 4; ++k) d2 += (pg[k] - pos[k])*(pg[k] - pos[k]); |
| 3806 | dist2[2*j+0] = d2; |
| 3807 | dist2[2*j+1] = j; |
| 3808 | } |
| 3809 | qsort(dist2, grid_size, 2*sizeof(int), iq3_compare_func); |
| 3810 | int local_counter = offsets[i]; |
| 3811 | kmap_q3xs[i] = -(local_counter + 1); |
| 3812 | int d2 = dist2[0]; |
| 3813 | uint16_t * start = &kneighbors_q3xs[local_counter++]; |
| 3814 | int n = 0, nhave = 1; |
| 3815 | for (int j = 0; j < grid_size; ++j) { |
| 3816 | if (dist2[2*j] > d2) { |
| 3817 | if (nhave == nwant) break; |
| 3818 | d2 = dist2[2*j]; |
| 3819 | ++nhave; |
| 3820 | } |
| 3821 | kneighbors_q3xs[local_counter++] = dist2[2*j+1]; |
| 3822 | ++n; |
| 3823 | } |
| 3824 | *start = n; |
| 3825 | } |
| 3826 | free(dist2); |
| 3827 | } |
| 3828 | free(offsets); |
| 3829 | free(n_per_i); |
| 3830 | } |
| 3831 | |
| 3832 | void iq3xs_free_impl(int grid_size) { |
| 3833 | GGML_ASSERT(grid_size == 256 || grid_size == 512)if (!(grid_size == 256 || grid_size == 512)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3833, "GGML_ASSERT(%s) failed", "grid_size == 256 || grid_size == 512" ); |
| 3834 | const int gindex = iq3_data_index(grid_size); |
| 3835 | if (iq3_data[gindex].grid) { |
| 3836 | free(iq3_data[gindex].grid); iq3_data[gindex].grid = NULL((void*)0); |
| 3837 | free(iq3_data[gindex].map); iq3_data[gindex].map = NULL((void*)0); |
| 3838 | free(iq3_data[gindex].neighbours); iq3_data[gindex].neighbours = NULL((void*)0); |
| 3839 | } |
| 3840 | } |
| 3841 | |
| 3842 | static int iq3_find_best_neighbour(const uint16_t * GGML_RESTRICTrestrict neighbours, const uint32_t * GGML_RESTRICTrestrict grid, |
| 3843 | const float * GGML_RESTRICTrestrict xval, const float * GGML_RESTRICTrestrict weight, float scale, int8_t * GGML_RESTRICTrestrict L) { |
| 3844 | int num_neighbors = neighbours[0]; |
| 3845 | GGML_ASSERT(num_neighbors > 0)if (!(num_neighbors > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3845, "GGML_ASSERT(%s) failed", "num_neighbors > 0"); |
| 3846 | float best_d2 = FLT_MAX3.40282347e+38F; |
| 3847 | int grid_index = -1; |
| 3848 | for (int j = 1; j <= num_neighbors; ++j) { |
| 3849 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 3850 | float d2 = 0; |
| 3851 | for (int i = 0; i < 4; ++i) { |
| 3852 | float q = pg[i]; |
| 3853 | float diff = scale*q - xval[i]; |
| 3854 | d2 += weight[i]*diff*diff; |
| 3855 | } |
| 3856 | if (d2 < best_d2) { |
| 3857 | best_d2 = d2; grid_index = neighbours[j]; |
| 3858 | } |
| 3859 | } |
| 3860 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3860, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 3861 | const int8_t * pg = (const int8_t *)(grid + grid_index); |
| 3862 | for (int i = 0; i < 4; ++i) L[i] = (pg[i] - 1)/2; |
| 3863 | return grid_index; |
| 3864 | } |
| 3865 | |
| 3866 | static void quantize_row_iq3_xxs_impl(int grid_size, const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, |
| 3867 | const float * GGML_RESTRICTrestrict quant_weights) { |
| 3868 | |
| 3869 | const int gindex = iq3_data_index(grid_size); |
| 3870 | |
| 3871 | const uint32_t * kgrid_q3xs = iq3_data[gindex].grid; |
| 3872 | const int * kmap_q3xs = iq3_data[gindex].map; |
| 3873 | const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours; |
| 3874 | |
| 3875 | //GGML_ASSERT(quant_weights && "missing quantization weights"); |
| 3876 | GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3876, "GGML_ASSERT(%s) failed", "kgrid_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3877 | GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3877, "GGML_ASSERT(%s) failed", "kmap_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3878 | GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3878, "GGML_ASSERT(%s) failed", "kneighbors_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 3879 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 3879, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 3880 | |
| 3881 | const int kMaxQ = 8; |
| 3882 | |
| 3883 | const int64_t nbl = n/QK_K256; |
| 3884 | |
| 3885 | ggml_fp16_t * dh; |
| 3886 | uint8_t * qs; |
| 3887 | int block_size; |
| 3888 | if (grid_size == 256) { |
| 3889 | block_iq3_xxs * y = vy; |
| 3890 | dh = &y->d; |
| 3891 | qs = y->qs; |
| 3892 | block_size = sizeof(block_iq3_xxs); |
| 3893 | } else { |
| 3894 | block_iq3_s * y = vy; |
| 3895 | dh = &y->d; |
| 3896 | qs = y->qs; |
| 3897 | block_size = sizeof(block_iq3_s); |
| 3898 | } |
| 3899 | int quant_size = block_size - sizeof(ggml_fp16_t); |
| 3900 | |
| 3901 | float scales[QK_K256/32]; |
| 3902 | float weight[32]; |
| 3903 | float xval[32]; |
| 3904 | int8_t L[32]; |
| 3905 | int8_t Laux[32]; |
| 3906 | float waux[32]; |
| 3907 | bool_Bool is_on_grid[8]; |
| 3908 | bool_Bool is_on_grid_aux[8]; |
| 3909 | uint8_t block_signs[8]; |
| 3910 | uint8_t q3[3*(QK_K256/8)+QK_K256/32]; |
| 3911 | uint32_t * scales_and_signs = (uint32_t *)(q3 + QK_K256/4); |
| 3912 | uint8_t * qh = q3 + 3*(QK_K256/8); |
| 3913 | |
| 3914 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 3915 | |
| 3916 | dh[0] = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 3917 | memset(q3, 0, 3*QK_K256/8+QK_K256/32); |
| 3918 | |
| 3919 | float max_scale = 0; |
| 3920 | |
| 3921 | const float * xbl = x + QK_K256*ibl; |
| 3922 | float sumx2 = 0; |
| 3923 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 3924 | float sigma2 = 2*sumx2/QK_K256; |
| 3925 | |
| 3926 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 3927 | const float * xb = xbl + 32*ib; |
| 3928 | if (quant_weights) { |
| 3929 | const float * qw = quant_weights + QK_K256*ibl + 32*ib; |
| 3930 | for (int i = 0; i < 32; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 3931 | } else { |
| 3932 | for (int i = 0; i < 32; ++i) weight[i] = xb[i]*xb[i]; |
| 3933 | } |
| 3934 | for (int i = 0; i < 32; ++i) waux[i] = sqrtf(weight[i]); |
| 3935 | for (int k = 0; k < 4; ++k) { |
| 3936 | int nflip = 0; |
| 3937 | uint8_t s = 0; |
| 3938 | for (int i = 0; i < 8; ++i) { |
| 3939 | if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; |
| 3940 | else { |
| 3941 | xval[8*k + i] = -xb[8*k + i]; ++nflip; s |= (1 << i); |
| 3942 | } |
| 3943 | } |
| 3944 | if (nflip%2) { |
| 3945 | int imin = 0; float min = weight[8*k+imin]*xb[8*k+imin]*xb[8*k+imin]; |
| 3946 | for (int i = 1; i < 8; ++i) { |
| 3947 | float ax = weight[8*k+i]*xb[8*k+i]*xb[8*k+i]; |
| 3948 | if (ax < min) { |
| 3949 | min = ax; imin = i; |
| 3950 | } |
| 3951 | } |
| 3952 | xval[8*k+imin] = -xval[8*k+imin]; |
| 3953 | s ^= (1 << imin); |
| 3954 | } |
| 3955 | block_signs[k] = s & 127; |
| 3956 | } |
| 3957 | float max = xval[0]; |
| 3958 | for (int i = 1; i < 32; ++i) max = MAX(max, xval[i])((max) > (xval[i]) ? (max) : (xval[i])); |
| 3959 | memset(L, 0, 32); |
| 3960 | if (max < GROUP_MAX_EPS_IQ3_XXS1e-8f) { |
| 3961 | scales[ib] = 0; |
| 3962 | continue; |
| 3963 | } |
| 3964 | float best = 0; |
| 3965 | float scale = max/(2*kMaxQ-1); |
| 3966 | for (int k = 0; k < 8; ++k) is_on_grid[k] = true1; |
| 3967 | for (int is = -15; is <= 15; ++is) { |
| 3968 | float id = (2*kMaxQ-1+is*0.2f)/max; |
| 3969 | float this_scale = 1/id; |
| 3970 | for (int k = 0; k < 8; ++k) { |
| 3971 | for (int i = 0; i < 4; ++i) { |
| 3972 | int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); |
| 3973 | Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 3974 | } |
| 3975 | uint16_t u = 0; |
| 3976 | for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i); |
| 3977 | int grid_index = kmap_q3xs[u]; |
| 3978 | is_on_grid_aux[k] = true1; |
| 3979 | if (grid_index < 0) { |
| 3980 | is_on_grid_aux[k] = false0; |
| 3981 | const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; |
| 3982 | grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k); |
| 3983 | } |
| 3984 | } |
| 3985 | float sumqx = 0, sumq2 = 0; |
| 3986 | for (int i = 0; i < 32; ++i) { |
| 3987 | float w = weight[i]; |
| 3988 | float q = 2*Laux[i] + 1; |
| 3989 | sumqx += w*xval[i]*q; |
| 3990 | sumq2 += w*q*q; |
| 3991 | } |
| 3992 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 3993 | scale = sumqx/sumq2; best = scale*sumqx; |
| 3994 | for (int i = 0; i < 32; ++i) L[i] = Laux[i]; |
| 3995 | for (int k = 0; k < 8; ++k) is_on_grid[k] = is_on_grid_aux[k]; |
| 3996 | } |
| 3997 | } |
| 3998 | int n_not_ongrid = 0; |
| 3999 | for (int k = 0; k < 8; ++k) if (!is_on_grid[k]) ++n_not_ongrid; |
| 4000 | if (n_not_ongrid > 0 && scale > 0) { |
| 4001 | float id = 1/scale; |
| 4002 | for (int k = 0; k < 8; ++k) { |
| 4003 | if (is_on_grid[k]) continue; |
| 4004 | uint16_t u = 0; |
| 4005 | for (int i = 0; i < 4; ++i) { |
| 4006 | int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); |
| 4007 | l = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 4008 | u |= (l << 3*i); |
| 4009 | } |
| 4010 | int grid_index = kmap_q3xs[u]; |
| 4011 | if (grid_index < 0) { |
| 4012 | const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; |
| 4013 | grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k); |
| 4014 | } |
| 4015 | const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index); |
| 4016 | for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2; |
| 4017 | } |
| 4018 | float sumqx = 0, sumq2 = 0; |
| 4019 | for (int i = 0; i < 32; ++i) { |
| 4020 | float w = weight[i]; |
| 4021 | float q = 2*L[i] + 1; |
| 4022 | sumqx += w*xval[i]*q; |
| 4023 | sumq2 += w*q*q; |
| 4024 | } |
| 4025 | if (sumq2 > 0) scale = sumqx/sumq2; |
| 4026 | } |
| 4027 | if (scale < 0) { |
| 4028 | // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale) |
| 4029 | // and correspondingly flip quant signs. |
| 4030 | scale = -scale; |
| 4031 | for (int k = 0; k < 4; ++k) block_signs[k] = (~block_signs[k]) & 127; |
| 4032 | } |
| 4033 | for (int k = 0; k < 8; ++k) { |
| 4034 | uint16_t u = 0; |
| 4035 | for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i); |
| 4036 | int grid_index = kmap_q3xs[u]; |
| 4037 | if (grid_index < 0) { |
| 4038 | printf("Oops: found point %u not on grid:", u); |
| 4039 | for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]); |
| 4040 | printf("\n"); |
| 4041 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4041, "fatal error"); |
| 4042 | } |
| 4043 | if (grid_size == 256) { |
| 4044 | q3[8*ib+k] = grid_index; |
| 4045 | } else { |
| 4046 | q3[8*ib+k] = grid_index & 255; |
| 4047 | qh[ib] |= ((grid_index >> 8) << k); |
| 4048 | } |
| 4049 | |
| 4050 | } |
| 4051 | scales_and_signs[ib] = block_signs[0] | (block_signs[1] << 7) | (block_signs[2] << 14) | (block_signs[3] << 21); |
| 4052 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4052, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 4053 | scales[ib] = scale; |
| 4054 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 4055 | } |
| 4056 | |
| 4057 | if (!max_scale) { |
| 4058 | memset(qs, 0, quant_size); |
| 4059 | dh += block_size/sizeof(ggml_fp16_t); |
| 4060 | qs += block_size; |
| 4061 | continue; |
| 4062 | } |
| 4063 | |
| 4064 | float d = max_scale/31; |
| 4065 | dh[0] = GGML_FP32_TO_FP16(d * 1.0125f)ggml_compute_fp32_to_fp16(d * 1.0125f); // small improvement via this fudge factor |
| 4066 | float id = 1/d; |
| 4067 | for (int ib = 0; ib < QK_K256/32; ++ib) { |
| 4068 | int l = nearest_int(0.5f*(id*scales[ib]-1)); |
| 4069 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 4070 | scales_and_signs[ib] |= ((uint32_t)l << 28); |
| 4071 | } |
| 4072 | memcpy(qs, q3, quant_size); |
| 4073 | |
| 4074 | dh += block_size/sizeof(ggml_fp16_t); |
| 4075 | qs += block_size; |
| 4076 | |
| 4077 | } |
| 4078 | } |
| 4079 | |
| 4080 | size_t quantize_iq3_xxs(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 4081 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4081, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 4082 | int64_t nblock = n_per_row/QK_K256; |
| 4083 | char * qrow = (char *)dst; |
| 4084 | for (int64_t row = 0; row < nrow; ++row) { |
| 4085 | quantize_row_iq3_xxs_impl(256, src, qrow, n_per_row, quant_weights); |
| 4086 | src += n_per_row; |
| 4087 | qrow += nblock*sizeof(block_iq3_xxs); |
| 4088 | } |
| 4089 | return nrow * nblock * sizeof(block_iq3_xxs); |
| 4090 | } |
| 4091 | |
| 4092 | void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICTrestrict x, block_iq3_xxs * GGML_RESTRICTrestrict y, int64_t k) { |
| 4093 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4093, __extension__ __PRETTY_FUNCTION__); })); |
| 4094 | quantize_row_iq3_xxs_impl(256, x, y, k, NULL((void*)0)); |
| 4095 | } |
| 4096 | |
| 4097 | static void quantize_row_iq3_s_impl(int block_size, const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int n, |
| 4098 | const float * GGML_RESTRICTrestrict quant_weights, |
| 4099 | float * scales, |
| 4100 | float * weight, |
| 4101 | float * xval, |
| 4102 | int8_t * L, |
| 4103 | int8_t * Laux, |
| 4104 | float * waux, |
| 4105 | bool_Bool * is_on_grid, |
| 4106 | bool_Bool * is_on_grid_aux, |
| 4107 | uint8_t * block_signs) { |
| 4108 | |
| 4109 | const int gindex = iq3_data_index(512); |
| 4110 | |
| 4111 | const uint32_t * kgrid_q3xs = iq3_data[gindex].grid; |
| 4112 | const int * kmap_q3xs = iq3_data[gindex].map; |
| 4113 | const uint16_t * kneighbors_q3xs = iq3_data[gindex].neighbours; |
| 4114 | |
| 4115 | //GGML_ASSERT(quant_weights && "missing quantization weights"); |
| 4116 | GGML_ASSERT(kgrid_q3xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4116, "GGML_ASSERT(%s) failed", "kgrid_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4117 | GGML_ASSERT(kmap_q3xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4117, "GGML_ASSERT(%s) failed", "kmap_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4118 | GGML_ASSERT(kneighbors_q3xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q3xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4118, "GGML_ASSERT(%s) failed", "kneighbors_q3xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4119 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4119, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 4120 | |
| 4121 | const int kMaxQ = 8; |
| 4122 | |
| 4123 | const int64_t nbl = n/QK_K256; |
| 4124 | |
| 4125 | block_iq3_s * y = vy; |
| 4126 | |
| 4127 | const int bs4 = block_size/4; |
| 4128 | const int bs8 = block_size/8; |
| 4129 | |
| 4130 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 4131 | |
| 4132 | memset(&y[ibl], 0, sizeof(block_iq3_s)); |
| 4133 | y[ibl].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 4134 | |
| 4135 | uint8_t * qs = y[ibl].qs; |
| 4136 | uint8_t * qh = y[ibl].qh; |
| 4137 | uint8_t * signs = y[ibl].signs; |
| 4138 | |
| 4139 | float max_scale = 0; |
| 4140 | |
| 4141 | const float * xbl = x + QK_K256*ibl; |
| 4142 | float sumx2 = 0; |
| 4143 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 4144 | float sigma2 = 2*sumx2/QK_K256; |
| 4145 | |
| 4146 | for (int ib = 0; ib < QK_K256/block_size; ++ib) { |
| 4147 | const float * xb = xbl + block_size*ib; |
| 4148 | if (quant_weights) { |
| 4149 | const float * qw = quant_weights + QK_K256*ibl + block_size*ib; |
| 4150 | for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 4151 | } else { |
| 4152 | for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i]; |
| 4153 | } |
| 4154 | for (int i = 0; i < block_size; ++i) waux[i] = sqrtf(weight[i]); |
| 4155 | for (int k = 0; k < bs8; ++k) { |
| 4156 | uint8_t s = 0; |
| 4157 | for (int i = 0; i < 8; ++i) { |
| 4158 | if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; |
| 4159 | else { |
| 4160 | xval[8*k + i] = -xb[8*k + i]; s |= (1 << i); |
| 4161 | } |
| 4162 | } |
| 4163 | block_signs[k] = s; |
| 4164 | } |
| 4165 | float max = xval[0]; |
| 4166 | for (int i = 1; i < block_size; ++i) max = MAX(max, xval[i])((max) > (xval[i]) ? (max) : (xval[i])); |
| 4167 | memset(L, 0, block_size); |
| 4168 | if (!max) { |
| 4169 | scales[ib] = 0; |
| 4170 | continue; |
| 4171 | } |
| 4172 | float best = 0; |
| 4173 | float scale = max/(2*kMaxQ-1); |
| 4174 | for (int k = 0; k < bs4; ++k) is_on_grid[k] = false0; |
| 4175 | for (int is = -9; is <= 9; ++is) { |
| 4176 | float id = (2*kMaxQ-1+is*0.2f)/max; |
| 4177 | float this_scale = 1/id; |
| 4178 | for (int k = 0; k < bs4; ++k) { |
| 4179 | for (int i = 0; i < 4; ++i) { |
| 4180 | int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); |
| 4181 | Laux[4*k+i] = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 4182 | } |
| 4183 | uint16_t u = 0; |
| 4184 | for (int i = 0; i < 4; ++i) u |= (Laux[4*k+i] << 3*i); |
| 4185 | int grid_index = kmap_q3xs[u]; |
| 4186 | is_on_grid_aux[k] = true1; |
| 4187 | if (grid_index < 0) { |
| 4188 | is_on_grid_aux[k] = false0; |
| 4189 | const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; |
| 4190 | grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, this_scale, Laux + 4*k); |
| 4191 | } |
| 4192 | } |
| 4193 | float sumqx = 0, sumq2 = 0; |
| 4194 | for (int i = 0; i < block_size; ++i) { |
| 4195 | float w = weight[i]; |
| 4196 | float q = 2*Laux[i] + 1; |
| 4197 | sumqx += w*xval[i]*q; |
| 4198 | sumq2 += w*q*q; |
| 4199 | } |
| 4200 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 4201 | scale = sumqx/sumq2; best = scale*sumqx; |
| 4202 | for (int i = 0; i < block_size; ++i) L[i] = Laux[i]; |
| 4203 | for (int k = 0; k < bs4; ++k) is_on_grid[k] = is_on_grid_aux[k]; |
| 4204 | } |
| 4205 | } |
| 4206 | int n_not_ongrid = 0; |
| 4207 | for (int k = 0; k < bs4; ++k) if (!is_on_grid[k]) ++n_not_ongrid; |
| 4208 | if (n_not_ongrid > 0 && scale > 0) { |
| 4209 | float id = 1/scale; |
| 4210 | for (int k = 0; k < bs4; ++k) { |
| 4211 | //if (is_on_grid[k]) continue; |
| 4212 | uint16_t u = 0; |
| 4213 | for (int i = 0; i < 4; ++i) { |
| 4214 | int l = nearest_int(0.5f*(id*xval[4*k+i]-1)); |
| 4215 | l = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 4216 | u |= (l << 3*i); |
| 4217 | } |
| 4218 | int grid_index = kmap_q3xs[u]; |
| 4219 | if (grid_index < 0) { |
| 4220 | const uint16_t * neighbours = kneighbors_q3xs - kmap_q3xs[u] - 1; |
| 4221 | grid_index = iq3_find_best_neighbour(neighbours, kgrid_q3xs, xval + 4*k, waux + 4*k, scale, L + 4*k); |
| 4222 | } |
| 4223 | const int8_t * pg = (const int8_t *)(kgrid_q3xs + grid_index); |
| 4224 | for (int i = 0; i < 4; ++i) L[4*k+i] = (pg[i] - 1)/2; |
| 4225 | } |
| 4226 | float sumqx = 0, sumq2 = 0; |
| 4227 | for (int i = 0; i < block_size; ++i) { |
| 4228 | float w = weight[i]; |
| 4229 | float q = 2*L[i] + 1; |
| 4230 | sumqx += w*xval[i]*q; |
| 4231 | sumq2 += w*q*q; |
| 4232 | } |
| 4233 | if (sumq2 > 0) scale = sumqx/sumq2; |
| 4234 | } |
| 4235 | if (scale < 0) { |
| 4236 | // This should never happen, but just in case, flip scale so that it is positive (we use uint's to encode the scale) |
| 4237 | // and correspondingly flip quant signs. |
| 4238 | scale = -scale; |
| 4239 | for (int k = 0; k < bs8; ++k) block_signs[k] = ~block_signs[k]; |
| 4240 | } |
| 4241 | for (int k = 0; k < bs4; ++k) { |
| 4242 | uint16_t u = 0; |
| 4243 | for (int i = 0; i < 4; ++i) u |= (L[4*k+i] << 3*i); |
| 4244 | int grid_index = kmap_q3xs[u]; |
| 4245 | if (grid_index < 0) { |
| 4246 | printf("Oops: found point %u not on grid:", u); |
| 4247 | for (int i = 0; i < 4; ++i) printf(" %d", L[4*k+i]); |
| 4248 | printf("\n"); |
| 4249 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4249, "fatal error"); |
| 4250 | } |
| 4251 | qs[k] = grid_index & 255; |
| 4252 | qh[(ib*bs4+k)/8] |= ((grid_index >> 8) << ((ib*bs4+k)%8)); |
| 4253 | } |
| 4254 | qs += bs4; |
| 4255 | for (int k = 0; k < bs8; ++k) signs[k] = block_signs[k]; |
| 4256 | signs += bs8; |
| 4257 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4257, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 4258 | scales[ib] = scale; |
| 4259 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 4260 | } |
| 4261 | |
| 4262 | if (!max_scale) { |
| 4263 | continue; |
| 4264 | } |
| 4265 | |
| 4266 | float d = max_scale/31; |
| 4267 | y[ibl].d = GGML_FP32_TO_FP16(d * 1.033f)ggml_compute_fp32_to_fp16(d * 1.033f); |
| 4268 | float id = 1/d; |
| 4269 | for (int ib = 0; ib < QK_K256/block_size; ib += 2) { |
| 4270 | int l1 = nearest_int(0.5f*(id*scales[ib+0]-1)); |
| 4271 | l1 = MAX(0, MIN(15, l1))((0) > (((15) < (l1) ? (15) : (l1))) ? (0) : (((15) < (l1) ? (15) : (l1)))); |
| 4272 | int l2 = nearest_int(0.5f*(id*scales[ib+1]-1)); |
| 4273 | l2 = MAX(0, MIN(15, l2))((0) > (((15) < (l2) ? (15) : (l2))) ? (0) : (((15) < (l2) ? (15) : (l2)))); |
| 4274 | y[ibl].scales[ib/2] = l1 | (l2 << 4); |
| 4275 | } |
| 4276 | |
| 4277 | } |
| 4278 | } |
| 4279 | |
| 4280 | #define IQ3S_BLOCK_SIZE32 32 |
| 4281 | size_t quantize_iq3_s(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 4282 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4282, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 4283 | int64_t nblock = n_per_row/QK_K256; |
| 4284 | float scales[QK_K256/IQ3S_BLOCK_SIZE32]; |
| 4285 | float weight[IQ3S_BLOCK_SIZE32]; |
| 4286 | float xval[IQ3S_BLOCK_SIZE32]; |
| 4287 | int8_t L[IQ3S_BLOCK_SIZE32]; |
| 4288 | int8_t Laux[IQ3S_BLOCK_SIZE32]; |
| 4289 | float waux[IQ3S_BLOCK_SIZE32]; |
| 4290 | bool_Bool is_on_grid[IQ3S_BLOCK_SIZE32/4]; |
| 4291 | bool_Bool is_on_grid_aux[IQ3S_BLOCK_SIZE32/4]; |
| 4292 | uint8_t block_signs[IQ3S_BLOCK_SIZE32/8]; |
| 4293 | char * qrow = (char *)dst; |
| 4294 | for (int64_t row = 0; row < nrow; ++row) { |
| 4295 | quantize_row_iq3_s_impl(IQ3S_BLOCK_SIZE32, src, qrow, n_per_row, quant_weights, |
| 4296 | scales, weight, xval, L, Laux, waux, is_on_grid, is_on_grid_aux, block_signs); |
| 4297 | src += n_per_row; |
| 4298 | qrow += nblock*sizeof(block_iq3_s); |
| 4299 | } |
| 4300 | return nrow * nblock * sizeof(block_iq3_s); |
| 4301 | } |
| 4302 | |
| 4303 | void quantize_row_iq3_s_ref(const float * GGML_RESTRICTrestrict x, block_iq3_s * GGML_RESTRICTrestrict y, int64_t k) { |
| 4304 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4304, __extension__ __PRETTY_FUNCTION__); })); |
| 4305 | quantize_iq3_s(x, y, 1, k, NULL((void*)0)); |
| 4306 | } |
| 4307 | |
| 4308 | |
| 4309 | // =================================== 1.5 bpw =================================================== |
| 4310 | |
| 4311 | static int iq1_find_best_neighbour(const uint16_t * GGML_RESTRICTrestrict neighbours, const uint64_t * GGML_RESTRICTrestrict grid, |
| 4312 | const float * GGML_RESTRICTrestrict xval, const float * GGML_RESTRICTrestrict weight, float * scale, int8_t * GGML_RESTRICTrestrict L, int ngrid) { |
| 4313 | int num_neighbors = neighbours[0]; |
| 4314 | GGML_ASSERT(num_neighbors > 0)if (!(num_neighbors > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4314, "GGML_ASSERT(%s) failed", "num_neighbors > 0"); |
| 4315 | float best_score = -FLT_MAX3.40282347e+38F; |
| 4316 | int grid_index = -1; |
| 4317 | for (int j = 1; j <= num_neighbors; ++j) { |
| 4318 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 4319 | float sumqx = 0, sumq2 = 0; |
| 4320 | for (int i = 0; i < 8; ++i) { |
| 4321 | float q = (pg[i] - 3)/2; |
| 4322 | float w = weight[i]; |
| 4323 | sumqx += w*q*xval[i]; |
| 4324 | sumq2 += w*q*q; |
| 4325 | } |
| 4326 | if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { |
| 4327 | *scale = sumqx/sumq2; best_score = *scale * sumqx; |
| 4328 | grid_index = neighbours[j]; |
| 4329 | } |
| 4330 | } |
| 4331 | if (grid_index < 0) { |
| 4332 | for (int i = 0; i < ngrid; ++i) { |
| 4333 | const int8_t * grid_i = (const int8_t *)(grid + i); |
| 4334 | float sumqx = 0, sumq2 = 0; |
| 4335 | for (int j = 0; j < 8; ++j) { |
| 4336 | float w = weight[j]; |
| 4337 | float q = (grid_i[j] - 3)/2; |
| 4338 | sumqx += w*q*xval[j]; |
| 4339 | sumq2 += w*q*q; |
| 4340 | } |
| 4341 | if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { |
| 4342 | *scale = sumqx/sumq2; best_score = *scale*sumqx; |
| 4343 | grid_index = i; |
| 4344 | } |
| 4345 | } |
| 4346 | } |
| 4347 | if (grid_index < 0) { |
| 4348 | printf("Oops, did not find grid point\n"); |
| 4349 | printf("Have %d neighbours\n", num_neighbors); |
| 4350 | for (int j = 1; j <= num_neighbors; ++j) { |
| 4351 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 4352 | float sumqx = 0, sumq2 = 0; |
| 4353 | for (int i = 0; i < 8; ++i) { |
| 4354 | float q = (pg[i] - 3)/2; |
| 4355 | float w = weight[i]; |
| 4356 | sumqx += w*q*xval[i]; |
| 4357 | sumq2 += w*q*q; |
| 4358 | } |
| 4359 | printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); |
| 4360 | } |
| 4361 | } |
| 4362 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4362, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 4363 | //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! |
| 4364 | *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result. |
| 4365 | //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! |
| 4366 | const int8_t * pg = (const int8_t *)(grid + grid_index); |
| 4367 | for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; |
| 4368 | return grid_index; |
| 4369 | } |
| 4370 | |
| 4371 | static int iq1_find_best_neighbour2(const uint16_t * GGML_RESTRICTrestrict neighbours, const uint64_t * GGML_RESTRICTrestrict grid, |
| 4372 | const float * GGML_RESTRICTrestrict xval, const float * GGML_RESTRICTrestrict weight, float scale, const float * GGML_RESTRICTrestrict xg, int8_t * GGML_RESTRICTrestrict L, int ngrid) { |
| 4373 | int num_neighbors = neighbours[0]; |
| 4374 | GGML_ASSERT(num_neighbors > 0)if (!(num_neighbors > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4374, "GGML_ASSERT(%s) failed", "num_neighbors > 0"); |
| 4375 | float best_score = FLT_MAX3.40282347e+38F; |
| 4376 | int grid_index = -1; |
| 4377 | for (int j = 1; j <= num_neighbors; ++j) { |
| 4378 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 4379 | float d2 = 0; |
| 4380 | for (int i = 0; i < 8; ++i) { |
| 4381 | float q = xg[(pg[i] - 1)/2]; |
| 4382 | float w = weight[i]; |
| 4383 | float diff = scale*q - xval[i]; |
| 4384 | d2 += w*diff*diff; |
| 4385 | } |
| 4386 | if (d2 < best_score) { |
| 4387 | best_score = d2; |
| 4388 | grid_index = neighbours[j]; |
| 4389 | } |
| 4390 | } |
| 4391 | if (grid_index < 0) { |
| 4392 | for (int i = 0; i < ngrid; ++i) { |
| 4393 | const int8_t * grid_i = (const int8_t *)(grid + i); |
| 4394 | float d2 = 0; |
| 4395 | for (int j = 0; j < 8; ++j) { |
| 4396 | float w = weight[j]; |
| 4397 | float q = xg[(grid_i[j] - 1)/2]; |
| 4398 | float diff = scale*q - xval[i]; |
| 4399 | d2 += w*diff*diff; |
| 4400 | } |
| 4401 | if (d2 < best_score) { |
| 4402 | best_score = d2; |
| 4403 | grid_index = i; |
| 4404 | } |
| 4405 | } |
| 4406 | } |
| 4407 | if (grid_index < 0) { |
| 4408 | printf("Oops, did not find grid point\n"); |
| 4409 | printf("Have %d neighbours\n", num_neighbors); |
| 4410 | for (int j = 1; j <= num_neighbors; ++j) { |
| 4411 | const int8_t * pg = (const int8_t *)(grid + neighbours[j]); |
| 4412 | float sumqx = 0, sumq2 = 0; |
| 4413 | for (int i = 0; i < 8; ++i) { |
| 4414 | float q = xg[(pg[i] - 1)/2]; |
| 4415 | float w = weight[i]; |
| 4416 | sumqx += w*q*xval[i]; |
| 4417 | sumq2 += w*q*q; |
| 4418 | } |
| 4419 | printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); |
| 4420 | } |
| 4421 | } |
| 4422 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4422, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 4423 | const int8_t * pg = (const int8_t *)(grid + grid_index); |
| 4424 | for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; |
| 4425 | return grid_index; |
| 4426 | } |
| 4427 | |
| 4428 | static int iq1_sort_helper(const void * left, const void * right) { |
| 4429 | const float * l = left; |
| 4430 | const float * r = right; |
| 4431 | return *l < *r ? -1 : *l > *r ? 1 : 0; |
| 4432 | } |
| 4433 | |
| 4434 | #define IQ1S_BLOCK_SIZE32 32 |
| 4435 | #define IQ1M_BLOCK_SIZE16 16 |
| 4436 | static void quantize_row_iq1_s_impl(const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, const float * GGML_RESTRICTrestrict quant_weights, |
| 4437 | float * scales, |
| 4438 | float * weight, |
| 4439 | float * sumx, |
| 4440 | float * sumw, |
| 4441 | float * pairs, |
| 4442 | int8_t * L, |
| 4443 | uint16_t * index, |
| 4444 | int8_t * shifts) { |
| 4445 | |
| 4446 | const int gindex = iq2_data_index(GGML_TYPE_IQ1_S); |
| 4447 | |
| 4448 | const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; |
| 4449 | const int * kmap_q2xs = iq2_data[gindex].map; |
| 4450 | const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; |
| 4451 | |
| 4452 | GGML_ASSERT(quant_weights && "missing quantization weights")if (!(quant_weights && "missing quantization weights" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4452, "GGML_ASSERT(%s) failed", "quant_weights && \"missing quantization weights\"" ); |
| 4453 | GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4453, "GGML_ASSERT(%s) failed", "kgrid_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4454 | GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4454, "GGML_ASSERT(%s) failed", "kmap_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4455 | GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4455, "GGML_ASSERT(%s) failed", "kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4456 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4456, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 4457 | |
| 4458 | block_iq1_s * y = vy; |
| 4459 | |
| 4460 | const int64_t nbl = n/QK_K256; |
| 4461 | |
| 4462 | const int block_size = IQ1S_BLOCK_SIZE32; |
| 4463 | |
| 4464 | const float x_p[3] = {-1 + IQ1S_DELTA0.125f, IQ1S_DELTA0.125f, 1 + IQ1S_DELTA0.125f}; |
| 4465 | const float x_m[3] = {-1 - IQ1S_DELTA0.125f, -IQ1S_DELTA0.125f, 1 - IQ1S_DELTA0.125f}; |
| 4466 | |
| 4467 | |
| 4468 | int * idx = (int *)(pairs + 1); |
| 4469 | |
| 4470 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 4471 | |
| 4472 | y[ibl].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 4473 | memset(y[ibl].qs, 0, QK_K256/8); |
| 4474 | memset(y[ibl].qh, 0, QK_K256/16); |
| 4475 | |
| 4476 | float max_scale = 0; |
| 4477 | |
| 4478 | const float * xbl = x + QK_K256*ibl; |
| 4479 | float sumx2 = 0; |
| 4480 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 4481 | float sigma2 = 2*sumx2/QK_K256; |
| 4482 | |
| 4483 | for (int ib = 0; ib < QK_K256/block_size; ++ib) { |
| 4484 | const float * xb = xbl + block_size*ib; |
| 4485 | const float * qw = quant_weights + QK_K256*ibl + block_size*ib; |
| 4486 | for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 4487 | float max = fabsf(xb[0]); |
| 4488 | for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]))((max) > (fabsf(xb[i])) ? (max) : (fabsf(xb[i]))); |
| 4489 | if (max < GROUP_MAX_EPS_IQ1_S1e-12f) { |
| 4490 | scales[ib] = 0; |
| 4491 | shifts[ib] = 1; |
| 4492 | memset(L, 1, block_size); |
| 4493 | continue; |
| 4494 | } |
| 4495 | // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. |
| 4496 | // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two |
| 4497 | // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights |
| 4498 | // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and |
| 4499 | // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale |
| 4500 | // for each possible and score for each split. |
| 4501 | for (int j = 0; j < block_size; ++j) { |
| 4502 | pairs[2*j] = xb[j]; |
| 4503 | idx[2*j] = j; |
| 4504 | } |
| 4505 | qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper); |
| 4506 | { |
| 4507 | sumx[0] = sumw[0] = 0; |
| 4508 | for (int j = 0; j < block_size; ++j) { |
| 4509 | int i = idx[2*j]; |
| 4510 | sumx[j+1] = sumx[j] + weight[i]*xb[i]; |
| 4511 | sumw[j+1] = sumw[j] + weight[i]; |
| 4512 | } |
| 4513 | } |
| 4514 | float best_score = -FLT_MAX3.40282347e+38F, scale = max; |
| 4515 | int besti1 = -1, besti2 = -1, best_shift = 0; |
| 4516 | for (int i1 = 0; i1 <= block_size; ++i1) { |
| 4517 | for (int i2 = i1; i2 <= block_size; ++i2) { |
| 4518 | float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[block_size] - sumx[i2])*x_p[2]; |
| 4519 | float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[block_size] - sumw[i2])*x_p[2]*x_p[2]; |
| 4520 | if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { |
| 4521 | scale = sumqx/sumq2; best_score = scale*sumqx; |
| 4522 | besti1 = i1; besti2 = i2; best_shift = 1; |
| 4523 | } |
| 4524 | sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[block_size] - sumx[i2])*x_m[2]; |
| 4525 | sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[block_size] - sumw[i2])*x_m[2]*x_m[2]; |
| 4526 | if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { |
| 4527 | scale = sumqx/sumq2; best_score = scale*sumqx; |
| 4528 | besti1 = i1; besti2 = i2; best_shift = -1; |
| 4529 | } |
| 4530 | } |
| 4531 | } |
| 4532 | if (besti1 < 0 || besti2 < 0 || best_shift == 0) { |
| 4533 | scales[ib] = 0; |
| 4534 | shifts[ib] = 1; |
| 4535 | memset(L, 1, block_size); |
| 4536 | continue; |
| 4537 | } |
| 4538 | for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; |
| 4539 | for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; |
| 4540 | for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2; |
| 4541 | if (scale < 0) { |
| 4542 | for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j]; |
| 4543 | scale = -scale; best_shift = -best_shift; |
| 4544 | } |
| 4545 | bool_Bool all_on_grid = true1; |
| 4546 | const float * xx = best_shift == 1 ? x_p : x_m; |
| 4547 | for (int k = 0; k < block_size/8; ++k) { |
| 4548 | uint16_t u = 0; |
| 4549 | for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j); |
| 4550 | int grid_index = kmap_q2xs[u]; |
| 4551 | if (grid_index < 0) { |
| 4552 | all_on_grid = false0; |
| 4553 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 4554 | grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S2048); |
| 4555 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4555, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 4556 | } |
| 4557 | index[k] = grid_index; |
| 4558 | } |
| 4559 | if (!all_on_grid) { |
| 4560 | float sumqx = 0, sumq2 = 0; |
| 4561 | for (int k = 0; k < block_size/8; ++k) { |
| 4562 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]); |
| 4563 | for (int j = 0; j < 8; ++j) { |
| 4564 | float w = weight[8*k + j]; |
| 4565 | float q = xx[(pg[j] - 1)/2]; |
| 4566 | sumqx += w*q*xb[8*k+j]; |
| 4567 | sumq2 += w*q*q; |
| 4568 | } |
| 4569 | } |
| 4570 | if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2; |
| 4571 | } |
| 4572 | uint16_t h = 0; |
| 4573 | for (int k = 0; k < block_size/8; ++k) { |
| 4574 | y[ibl].qs[(block_size/8)*ib + k] = index[k] & 255; |
| 4575 | h |= (index[k] >> 8) << 3*k; |
| 4576 | } |
| 4577 | y[ibl].qh[ib] = h; |
| 4578 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4578, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 4579 | scales[ib] = scale; |
| 4580 | shifts[ib] = best_shift; |
| 4581 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 4582 | } |
| 4583 | |
| 4584 | if (!max_scale) { |
| 4585 | continue; |
| 4586 | } |
| 4587 | |
| 4588 | float d = max_scale/15; |
| 4589 | y[ibl].d = GGML_FP32_TO_FP16(d*1.125f)ggml_compute_fp32_to_fp16(d*1.125f); // 1.125f is another fudge factor. Don't ask me why it is needed. |
| 4590 | float id = 1/d; |
| 4591 | for (int ib = 0; ib < QK_K256/block_size; ++ib) { |
| 4592 | int l = nearest_int(0.5f*(id*scales[ib]-1)); |
| 4593 | l = MAX(0, MIN(7, l))((0) > (((7) < (l) ? (7) : (l))) ? (0) : (((7) < (l) ? (7) : (l)))); |
| 4594 | if (shifts[ib] == -1) l |= 8; |
| 4595 | y[ibl].qh[ib] |= (l << 12); |
| 4596 | } |
| 4597 | } |
| 4598 | } |
| 4599 | |
| 4600 | size_t quantize_iq1_s(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 4601 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4601, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 4602 | float scales[QK_K256/IQ1S_BLOCK_SIZE32]; |
| 4603 | float weight[IQ1S_BLOCK_SIZE32]; |
| 4604 | int8_t L[IQ1S_BLOCK_SIZE32]; |
| 4605 | float sumx[IQ1S_BLOCK_SIZE32+1]; |
| 4606 | float sumw[IQ1S_BLOCK_SIZE32+1]; |
| 4607 | float pairs[2*IQ1S_BLOCK_SIZE32]; |
| 4608 | uint16_t index[IQ1S_BLOCK_SIZE32/8]; |
| 4609 | int8_t shifts[QK_K256/IQ1S_BLOCK_SIZE32]; |
| 4610 | int64_t nblock = n_per_row/QK_K256; |
| 4611 | char * qrow = (char *)dst; |
| 4612 | for (int64_t row = 0; row < nrow; ++row) { |
| 4613 | quantize_row_iq1_s_impl(src, qrow, n_per_row, quant_weights, scales, weight, sumx, sumw, pairs, L, index, shifts); |
| 4614 | src += n_per_row; |
| 4615 | qrow += nblock*sizeof(block_iq1_s); |
| 4616 | } |
| 4617 | return nrow * nblock * sizeof(block_iq1_s); |
| 4618 | } |
| 4619 | |
| 4620 | static void quantize_row_iq1_m_impl(const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, const float * GGML_RESTRICTrestrict quant_weights, |
| 4621 | float * scales, |
| 4622 | float * weight, |
| 4623 | float * pairs, |
| 4624 | int8_t * L, |
| 4625 | uint16_t * index, |
| 4626 | int8_t * shifts) { |
| 4627 | |
| 4628 | const int gindex = iq2_data_index(GGML_TYPE_IQ1_M); |
| 4629 | |
| 4630 | const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; |
| 4631 | const int * kmap_q2xs = iq2_data[gindex].map; |
| 4632 | const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; |
| 4633 | |
| 4634 | //GGML_ASSERT(quant_weights && "missing quantization weights"); |
| 4635 | GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4635, "GGML_ASSERT(%s) failed", "kgrid_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4636 | GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4636, "GGML_ASSERT(%s) failed", "kmap_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4637 | GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4637, "GGML_ASSERT(%s) failed", "kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 4638 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4638, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 4639 | |
| 4640 | block_iq1_m * y = vy; |
| 4641 | |
| 4642 | const int64_t nbl = n/QK_K256; |
| 4643 | |
| 4644 | const int block_size = IQ1M_BLOCK_SIZE16; |
| 4645 | |
| 4646 | const float x_p[3] = {-1 + IQ1M_DELTA0.125f, IQ1M_DELTA0.125f, 1 + IQ1M_DELTA0.125f}; |
| 4647 | const float x_m[3] = {-1 - IQ1M_DELTA0.125f, -IQ1M_DELTA0.125f, 1 - IQ1M_DELTA0.125f}; |
| 4648 | const uint8_t masks[4] = {0x00, 0x80, 0x08, 0x88}; |
| 4649 | |
| 4650 | int * idx = (int *)(pairs + 1); |
| 4651 | |
| 4652 | float sumqx[4], sumq2[4]; |
| 4653 | |
| 4654 | iq1m_scale_t s; |
| 4655 | const float * xx; |
| 4656 | |
| 4657 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 4658 | memset(y[ibl].qs, 0, QK_K256/8); |
| 4659 | memset(y[ibl].qh, 0, QK_K256/16); |
| 4660 | memset(y[ibl].scales, 0, QK_K256/32); |
| 4661 | |
| 4662 | float max_scale = 0; |
| 4663 | |
| 4664 | const float * xbl = x + QK_K256*ibl; |
| 4665 | float sumx2 = 0; |
| 4666 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 4667 | float sigma2 = 2*sumx2/QK_K256; |
| 4668 | |
| 4669 | for (int ib = 0; ib < QK_K256/block_size; ++ib) { |
| 4670 | const float * xb = xbl + block_size*ib; |
| 4671 | if (quant_weights) { |
| 4672 | const float * qw = quant_weights + QK_K256*ibl + block_size*ib; |
| 4673 | for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 4674 | } else { |
| 4675 | for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i]; |
| 4676 | } |
| 4677 | float max = fabsf(xb[0]); |
| 4678 | for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i]))((max) > (fabsf(xb[i])) ? (max) : (fabsf(xb[i]))); |
| 4679 | if (max < GROUP_MAX_EPS_IQ1_M1e-7f) { |
| 4680 | scales[ib] = 0; |
| 4681 | shifts[ib] = 0; |
| 4682 | memset(L, 1, block_size); |
| 4683 | continue; |
| 4684 | } |
| 4685 | // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. |
| 4686 | // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two |
| 4687 | // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights |
| 4688 | // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and |
| 4689 | // Wi = sum[weight[j], j = 0...i], and use these to quckly get get the optimum scale |
| 4690 | // for each possible and score for each split. |
| 4691 | for (int j = 0; j < block_size; ++j) { |
| 4692 | pairs[2*j] = xb[j]; |
| 4693 | idx[2*j] = j; |
| 4694 | } |
| 4695 | qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper); |
| 4696 | float best_score = -FLT_MAX3.40282347e+38F, scale = max; |
| 4697 | int besti1 = -1, besti2 = -1, best_k = -1; |
| 4698 | // 0: +, + |
| 4699 | // 1: +, - |
| 4700 | // 2: -, + |
| 4701 | // 3: -, - |
| 4702 | for (int i1 = 0; i1 <= block_size; ++i1) { |
| 4703 | for (int i2 = i1; i2 <= block_size; ++i2) { |
| 4704 | memset(sumqx, 0, 4*sizeof(float)); |
| 4705 | memset(sumq2, 0, 4*sizeof(float)); |
| 4706 | for (int j = 0; j < i1; ++j) { |
| 4707 | int i = idx[2*j]; |
| 4708 | if (i < block_size/2) { |
| 4709 | sumqx[0] += weight[i]*x_p[0]*xb[i]; |
| 4710 | sumqx[1] += weight[i]*x_p[0]*xb[i]; |
| 4711 | sumqx[2] += weight[i]*x_m[0]*xb[i]; |
| 4712 | sumqx[3] += weight[i]*x_m[0]*xb[i]; |
| 4713 | sumq2[0] += weight[i]*x_p[0]*x_p[0]; |
| 4714 | sumq2[1] += weight[i]*x_p[0]*x_p[0]; |
| 4715 | sumq2[2] += weight[i]*x_m[0]*x_m[0]; |
| 4716 | sumq2[3] += weight[i]*x_m[0]*x_m[0]; |
| 4717 | } else { |
| 4718 | sumqx[0] += weight[i]*x_p[0]*xb[i]; |
| 4719 | sumqx[2] += weight[i]*x_p[0]*xb[i]; |
| 4720 | sumqx[1] += weight[i]*x_m[0]*xb[i]; |
| 4721 | sumqx[3] += weight[i]*x_m[0]*xb[i]; |
| 4722 | sumq2[0] += weight[i]*x_p[0]*x_p[0]; |
| 4723 | sumq2[2] += weight[i]*x_p[0]*x_p[0]; |
| 4724 | sumq2[1] += weight[i]*x_m[0]*x_m[0]; |
| 4725 | sumq2[3] += weight[i]*x_m[0]*x_m[0]; |
| 4726 | } |
| 4727 | } |
| 4728 | for (int j = i1; j < i2; ++j) { |
| 4729 | int i = idx[2*j]; |
| 4730 | if (i < block_size/2) { |
| 4731 | sumqx[0] += weight[i]*x_p[1]*xb[i]; |
| 4732 | sumqx[1] += weight[i]*x_p[1]*xb[i]; |
| 4733 | sumqx[2] += weight[i]*x_m[1]*xb[i]; |
| 4734 | sumqx[3] += weight[i]*x_m[1]*xb[i]; |
| 4735 | sumq2[0] += weight[i]*x_p[1]*x_p[1]; |
| 4736 | sumq2[1] += weight[i]*x_p[1]*x_p[1]; |
| 4737 | sumq2[2] += weight[i]*x_m[1]*x_m[1]; |
| 4738 | sumq2[3] += weight[i]*x_m[1]*x_m[1]; |
| 4739 | } else { |
| 4740 | sumqx[0] += weight[i]*x_p[1]*xb[i]; |
| 4741 | sumqx[2] += weight[i]*x_p[1]*xb[i]; |
| 4742 | sumqx[1] += weight[i]*x_m[1]*xb[i]; |
| 4743 | sumqx[3] += weight[i]*x_m[1]*xb[i]; |
| 4744 | sumq2[0] += weight[i]*x_p[1]*x_p[1]; |
| 4745 | sumq2[2] += weight[i]*x_p[1]*x_p[1]; |
| 4746 | sumq2[1] += weight[i]*x_m[1]*x_m[1]; |
| 4747 | sumq2[3] += weight[i]*x_m[1]*x_m[1]; |
| 4748 | } |
| 4749 | } |
| 4750 | for (int j = i2; j < block_size; ++j) { |
| 4751 | int i = idx[2*j]; |
| 4752 | if (i < block_size/2) { |
| 4753 | sumqx[0] += weight[i]*x_p[2]*xb[i]; |
| 4754 | sumqx[1] += weight[i]*x_p[2]*xb[i]; |
| 4755 | sumqx[2] += weight[i]*x_m[2]*xb[i]; |
| 4756 | sumqx[3] += weight[i]*x_m[2]*xb[i]; |
| 4757 | sumq2[0] += weight[i]*x_p[2]*x_p[2]; |
| 4758 | sumq2[1] += weight[i]*x_p[2]*x_p[2]; |
| 4759 | sumq2[2] += weight[i]*x_m[2]*x_m[2]; |
| 4760 | sumq2[3] += weight[i]*x_m[2]*x_m[2]; |
| 4761 | } else { |
| 4762 | sumqx[0] += weight[i]*x_p[2]*xb[i]; |
| 4763 | sumqx[2] += weight[i]*x_p[2]*xb[i]; |
| 4764 | sumqx[1] += weight[i]*x_m[2]*xb[i]; |
| 4765 | sumqx[3] += weight[i]*x_m[2]*xb[i]; |
| 4766 | sumq2[0] += weight[i]*x_p[2]*x_p[2]; |
| 4767 | sumq2[2] += weight[i]*x_p[2]*x_p[2]; |
| 4768 | sumq2[1] += weight[i]*x_m[2]*x_m[2]; |
| 4769 | sumq2[3] += weight[i]*x_m[2]*x_m[2]; |
| 4770 | } |
| 4771 | } |
| 4772 | for (int k = 0; k < 4; ++k) { |
| 4773 | if (sumq2[k] > 0 && sumqx[k]*sumqx[k] > best_score*sumq2[k]) { |
| 4774 | scale = sumqx[k]/sumq2[k]; best_score = scale*sumqx[k]; |
| 4775 | besti1 = i1; besti2 = i2; best_k = k; |
| 4776 | } |
| 4777 | } |
| 4778 | } |
| 4779 | } |
| 4780 | if (besti1 < 0 || besti2 < 0 || best_k < 0) { |
| 4781 | scales[ib] = 0; |
| 4782 | shifts[ib] = 0; |
| 4783 | memset(L, 1, block_size); |
| 4784 | continue; |
| 4785 | } |
| 4786 | for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; |
| 4787 | for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; |
| 4788 | for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2; |
| 4789 | if (scale < 0) { |
| 4790 | for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j]; |
| 4791 | scale = -scale; |
| 4792 | best_k = best_k == 0 ? 3 : best_k == 1 ? 2 : best_k == 2 ? 1 : 0; |
| 4793 | } |
| 4794 | bool_Bool all_on_grid = true1; |
| 4795 | for (int k = 0; k < block_size/8; ++k) { |
| 4796 | if (k == 0) xx = best_k < 2 ? x_p : x_m; |
| 4797 | else xx = best_k%2 == 0 ? x_p : x_m; |
| 4798 | uint16_t u = 0; |
| 4799 | for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j); |
| 4800 | int grid_index = kmap_q2xs[u]; |
| 4801 | if (grid_index < 0) { |
| 4802 | all_on_grid = false0; |
| 4803 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 4804 | grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, NGRID_IQ1S2048); |
| 4805 | GGML_ASSERT(grid_index >= 0)if (!(grid_index >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4805, "GGML_ASSERT(%s) failed", "grid_index >= 0"); |
| 4806 | } |
| 4807 | index[k] = grid_index; |
| 4808 | } |
| 4809 | if (!all_on_grid) { |
| 4810 | float sumqx_f = 0, sumq2_f = 0; |
| 4811 | for (int k = 0; k < block_size/8; ++k) { |
| 4812 | if (k == 0) xx = best_k < 2 ? x_p : x_m; |
| 4813 | else xx = best_k%2 == 0 ? x_p : x_m; |
| 4814 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + index[k]); |
| 4815 | for (int j = 0; j < 8; ++j) { |
| 4816 | float w = weight[8*k + j]; |
| 4817 | float q = xx[(pg[j] - 1)/2]; |
| 4818 | sumqx_f += w*q*xb[8*k+j]; |
| 4819 | sumq2_f += w*q*q; |
| 4820 | } |
| 4821 | } |
| 4822 | if (sumqx_f > 0 && sumq2_f > 0) scale = sumqx_f/sumq2_f; |
| 4823 | } |
| 4824 | y[ibl].qs[2*ib + 0] = index[0] & 255; |
| 4825 | y[ibl].qs[2*ib + 1] = index[1] & 255; |
| 4826 | y[ibl].qh[ib] = (index[0] >> 8) | ((index[1] >> 8) << 4); |
| 4827 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4827, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 4828 | scales[ib] = scale; |
| 4829 | shifts[ib] = best_k; |
| 4830 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 4831 | } |
| 4832 | |
| 4833 | if (!max_scale) { |
| 4834 | continue; |
| 4835 | } |
| 4836 | |
| 4837 | uint16_t * sc = (uint16_t *)y[ibl].scales; |
| 4838 | float d = max_scale/15; |
| 4839 | float id = 1/d; |
| 4840 | float sumqx_f = 0, sumq2_f = 0; |
| 4841 | for (int ib = 0; ib < QK_K256/block_size; ++ib) { |
| 4842 | int l = nearest_int(0.5f*(id*scales[ib+0]-1)); |
| 4843 | l = MAX(0, MIN(7, l))((0) > (((7) < (l) ? (7) : (l))) ? (0) : (((7) < (l) ? (7) : (l)))); |
| 4844 | sc[ib/4] |= (l << 3*(ib%4)); |
| 4845 | y[ibl].qh[ib] |= masks[shifts[ib]]; |
| 4846 | const float * xb = xbl + block_size*ib; |
| 4847 | if (quant_weights) { |
| 4848 | const float * qw = quant_weights + QK_K256*ibl + block_size*ib; |
| 4849 | for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 4850 | } else { |
| 4851 | for (int i = 0; i < block_size; ++i) weight[i] = xb[i]*xb[i]; |
| 4852 | } |
| 4853 | for (int k = 0; k < block_size/8; ++k) { |
| 4854 | if (k == 0) xx = shifts[ib] < 2 ? x_p : x_m; |
| 4855 | else xx = shifts[ib]%2 == 0 ? x_p : x_m; |
| 4856 | const int8_t * pg = (const int8_t *)(kgrid_q2xs + y[ibl].qs[2*ib+k] + ((y[ibl].qh[ib] << (8 - 4*k)) & 0x700)); |
| 4857 | for (int j = 0; j < 8; ++j) { |
| 4858 | float w = weight[8*k + j]; |
| 4859 | float q = xx[(pg[j] - 1)/2]*(2*l+1); |
| 4860 | sumqx_f += w*q*xb[8*k+j]; |
| 4861 | sumq2_f += w*q*q; |
| 4862 | } |
| 4863 | } |
| 4864 | } |
| 4865 | if (sumq2_f > 0) d = sumqx_f/sumq2_f; |
| 4866 | s.f16 = GGML_FP32_TO_FP16(d*1.1125f)ggml_compute_fp32_to_fp16(d*1.1125f); // 1.1125f is another fudge factor. Don't ask me why it is needed. |
| 4867 | sc[0] |= ((s.u16 & 0x000f) << 12); |
| 4868 | sc[1] |= ((s.u16 & 0x00f0) << 8); |
| 4869 | sc[2] |= ((s.u16 & 0x0f00) << 4); |
| 4870 | sc[3] |= ((s.u16 & 0xf000) << 0); |
| 4871 | } |
| 4872 | } |
| 4873 | |
| 4874 | size_t quantize_iq1_m(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 4875 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 4875, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 4876 | float scales[QK_K256/IQ1M_BLOCK_SIZE16]; |
| 4877 | float weight[IQ1M_BLOCK_SIZE16]; |
| 4878 | int8_t L[IQ1M_BLOCK_SIZE16]; |
| 4879 | float pairs[2*IQ1M_BLOCK_SIZE16]; |
| 4880 | uint16_t index[IQ1M_BLOCK_SIZE16/8]; |
| 4881 | int8_t shifts[QK_K256/IQ1M_BLOCK_SIZE16]; |
| 4882 | int64_t nblock = n_per_row/QK_K256; |
| 4883 | char * qrow = (char *)dst; |
| 4884 | for (int64_t row = 0; row < nrow; ++row) { |
| 4885 | quantize_row_iq1_m_impl(src, qrow, n_per_row, quant_weights, scales, weight, pairs, L, index, shifts); |
| 4886 | src += n_per_row; |
| 4887 | qrow += nblock*sizeof(block_iq1_m); |
| 4888 | } |
| 4889 | return nrow * nblock * sizeof(block_iq1_m); |
| 4890 | } |
| 4891 | |
| 4892 | // ============================ 4-bit non-linear quants |
| 4893 | |
| 4894 | static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, const float * GGML_RESTRICTrestrict x, |
| 4895 | ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l, |
| 4896 | float * scales, float * weight, uint8_t * L, |
| 4897 | const int8_t * values, |
| 4898 | const float * quant_weights, |
| 4899 | const int ntry) { |
| 4900 | |
| 4901 | float sigma2 = 0; |
| 4902 | for (int j = 0; j < super_block_size; ++j) sigma2 += x[j]*x[j]; |
| 4903 | sigma2 *= 2.f/super_block_size; |
| 4904 | |
| 4905 | memset(q4, 0, super_block_size/2); |
| 4906 | dh[0] = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 4907 | |
| 4908 | float max_scale = 0, amax_scale = 0; |
| 4909 | for (int ib = 0; ib < super_block_size/block_size; ++ib) { |
| 4910 | const float * xb = x + ib*block_size; |
| 4911 | uint8_t * Lb = L + ib*block_size; |
| 4912 | if (quant_weights) { |
| 4913 | const float * qw = quant_weights + ib*block_size; |
| 4914 | for (int j = 0; j < block_size; ++j) weight[j] = qw[j] * sqrtf(sigma2 + xb[j]*xb[j]); |
| 4915 | } else { |
| 4916 | for (int j = 0; j < block_size; ++j) weight[j] = xb[j]*xb[j]; |
| 4917 | } |
| 4918 | float amax = 0, max = 0; |
| 4919 | for (int j = 0; j < block_size; ++j) { |
| 4920 | float ax = fabsf(xb[j]); |
| 4921 | if (ax > amax) { |
| 4922 | amax = ax; max = xb[j]; |
| 4923 | } |
| 4924 | } |
| 4925 | if (amax < GROUP_MAX_EPS1e-15f) { |
| 4926 | scales[ib] = 0; |
| 4927 | continue; |
| 4928 | } |
| 4929 | float d = ntry > 0 ? -max/values[0] : max/values[0]; |
| 4930 | float id = 1/d; |
| 4931 | float sumqx = 0, sumq2 = 0; |
| 4932 | for (int j = 0; j < block_size; ++j) { |
| 4933 | float al = id*xb[j]; |
| 4934 | int l = best_index_int8(16, values, al); |
| 4935 | Lb[j] = l; |
| 4936 | float q = values[l]; |
| 4937 | float w = weight[j]; |
| 4938 | sumqx += w*q*xb[j]; |
| 4939 | sumq2 += w*q*q; |
| 4940 | } |
| 4941 | d = sumq2 > 0 ? sumqx/sumq2 : 0.f; |
| 4942 | float best = d*sumqx; |
| 4943 | for (int itry = -ntry; itry <= ntry; ++itry) { |
| 4944 | id = (itry + values[0])/max; |
| 4945 | sumqx = sumq2 = 0; |
| 4946 | for (int j = 0; j < block_size; ++j) { |
| 4947 | float al = id*xb[j]; |
| 4948 | int l = best_index_int8(16, values, al); |
| 4949 | float q = values[l]; |
| 4950 | float w = weight[j]; |
| 4951 | sumqx += w*q*xb[j]; |
| 4952 | sumq2 += w*q*q; |
| 4953 | } |
| 4954 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 4955 | d = sumqx/sumq2; best = d * sumqx; |
| 4956 | } |
| 4957 | } |
| 4958 | scales[ib] = d; |
| 4959 | float abs_d = fabsf(d); |
| 4960 | if (abs_d > amax_scale) { |
| 4961 | amax_scale = abs_d; max_scale = d; |
| 4962 | } |
| 4963 | } |
| 4964 | |
| 4965 | if (super_block_size/block_size > 1) { |
| 4966 | int nb = super_block_size/block_size; |
| 4967 | memset(scales_h, 0, ((nb+7)/8)*sizeof(uint16_t)); |
| 4968 | float d = -max_scale/32; |
| 4969 | dh[0] = GGML_FP32_TO_FP16(d)ggml_compute_fp32_to_fp16(d); |
| 4970 | float id = d ? 1/d : 0.f; |
| 4971 | for (int ib = 0; ib < super_block_size/block_size; ++ib) { |
| 4972 | int l = nearest_int(id*scales[ib]); |
| 4973 | l = MAX(-32, MIN(31, l))((-32) > (((31) < (l) ? (31) : (l))) ? (-32) : (((31) < (l) ? (31) : (l)))); |
| 4974 | float dl = d * l; |
| 4975 | float idl = dl ? 1/dl : 0.f; |
| 4976 | uint8_t * Lb = L + ib*block_size; |
| 4977 | const float * xb = x + ib*block_size; |
| 4978 | for (int j = 0; j < block_size; ++j) { |
| 4979 | Lb[j] = best_index_int8(16, values, idl*xb[j]); |
| 4980 | } |
| 4981 | l += 32; |
| 4982 | uint8_t l_l = l & 0xf; |
| 4983 | uint8_t l_h = l >> 4; |
| 4984 | if (ib%2 == 0) scales_l[ib/2] = l_l; |
| 4985 | else scales_l[ib/2] |= (l_l << 4); |
| 4986 | scales_h[ib/8] |= (l_h << 2*(ib%8)); |
| 4987 | } |
| 4988 | } else { |
| 4989 | dh[0] = GGML_FP32_TO_FP16(scales[0])ggml_compute_fp32_to_fp16(scales[0]); |
| 4990 | if (ntry > 0) { |
| 4991 | float id = scales[0] ? 1/scales[0] : 0; |
| 4992 | for (int j = 0; j < super_block_size; ++j) { |
| 4993 | L[j] = best_index_int8(16, values, id*x[j]); |
| 4994 | } |
| 4995 | } |
| 4996 | } |
| 4997 | |
| 4998 | for (int i = 0; i < super_block_size/32; ++i) { |
| 4999 | for (int j = 0; j < 16; ++j) { |
| 5000 | q4[16*i + j] = L[32*i + j] | (L[32*i + 16 + j] << 4); |
| 5001 | } |
| 5002 | } |
| 5003 | } |
| 5004 | |
| 5005 | size_t quantize_iq4_nl(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 5006 | GGML_ASSERT(n_per_row%QK4_NL == 0)if (!(n_per_row%32 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5006, "GGML_ASSERT(%s) failed", "n_per_row%QK4_NL == 0"); |
| 5007 | int64_t nblock = n_per_row/QK4_NL32; |
| 5008 | char * qrow = (char *)dst; |
| 5009 | uint8_t L[QK4_NL32]; |
| 5010 | float weight[QK4_NL32]; |
| 5011 | uint16_t unused_h; |
| 5012 | uint8_t * unused_l = NULL((void*)0); |
| 5013 | float scale; |
| 5014 | for (int64_t row = 0; row < nrow; ++row) { |
| 5015 | block_iq4_nl * iq4 = (block_iq4_nl *)qrow; |
| 5016 | for (int ibl = 0; ibl < nblock; ++ibl) { |
| 5017 | const float * qw = quant_weights ? quant_weights + QK4_NL32*ibl : NULL((void*)0); |
| 5018 | quantize_row_iq4_nl_impl(QK4_NL32, 32, src + QK4_NL32*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l, |
| 5019 | &scale, weight, L, kvalues_iq4nl, qw, 7); |
| 5020 | } |
| 5021 | src += n_per_row; |
| 5022 | qrow += nblock*sizeof(block_iq4_nl); |
| 5023 | } |
| 5024 | return nrow * nblock * sizeof(block_iq4_nl); |
| 5025 | } |
| 5026 | |
| 5027 | //void quantize_row_iq4_nl_ref(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { |
| 5028 | void quantize_row_iq4_nl_ref(const float * GGML_RESTRICTrestrict x, block_iq4_nl * GGML_RESTRICTrestrict y, int64_t k) { |
| 5029 | GGML_ASSERT(k%QK4_NL == 0)if (!(k%32 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5029, "GGML_ASSERT(%s) failed", "k%QK4_NL == 0"); |
| 5030 | int64_t nblock = k/QK4_NL32; |
| 5031 | uint8_t L[QK4_NL32]; |
| 5032 | float weight[QK4_NL32]; |
| 5033 | uint16_t unused_h; |
| 5034 | uint8_t * unused_l = NULL((void*)0); |
| 5035 | float scale; |
| 5036 | block_iq4_nl * iq4 = y; |
| 5037 | for (int ibl = 0; ibl < nblock; ++ibl) { |
| 5038 | quantize_row_iq4_nl_impl(QK4_NL32, 32, x + QK4_NL32*ibl, &iq4[ibl].d, iq4[ibl].qs, &unused_h, unused_l, |
| 5039 | &scale, weight, L, kvalues_iq4nl, NULL((void*)0), -1); |
| 5040 | } |
| 5041 | } |
| 5042 | |
| 5043 | size_t quantize_iq4_xs(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 5044 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5044, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 5045 | int64_t nblock = n_per_row/QK_K256; |
| 5046 | char * qrow = (char *)dst; |
| 5047 | uint8_t L[QK_K256]; |
| 5048 | float weight[32]; |
| 5049 | float scales[QK_K256/32]; |
| 5050 | for (int64_t row = 0; row < nrow; ++row) { |
| 5051 | block_iq4_xs * iq4 = (block_iq4_xs *)qrow; |
| 5052 | for (int ibl = 0; ibl < nblock; ++ibl) { |
| 5053 | const float * qw = quant_weights ? quant_weights + QK_K256*ibl : NULL((void*)0); |
| 5054 | quantize_row_iq4_nl_impl(QK_K256, 32, src + QK_K256*ibl, &iq4[ibl].d, iq4[ibl].qs, &iq4[ibl].scales_h, iq4[ibl].scales_l, |
| 5055 | scales, weight, L, kvalues_iq4nl, qw, 7); |
| 5056 | } |
| 5057 | src += n_per_row; |
| 5058 | qrow += nblock*sizeof(block_iq4_xs); |
| 5059 | } |
| 5060 | return nrow * nblock * sizeof(block_iq4_xs); |
| 5061 | } |
| 5062 | |
| 5063 | void quantize_row_iq4_xs_ref(const float * GGML_RESTRICTrestrict x, block_iq4_xs * GGML_RESTRICTrestrict y, int64_t k) { |
| 5064 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5064, __extension__ __PRETTY_FUNCTION__); })); |
| 5065 | quantize_iq4_xs(x, y, 1, k, NULL((void*)0)); |
| 5066 | } |
| 5067 | |
| 5068 | // =============================== 2.5625 bpw |
| 5069 | |
| 5070 | static void quantize_row_iq2_s_impl(const float * GGML_RESTRICTrestrict x, void * GGML_RESTRICTrestrict vy, int64_t n, const float * GGML_RESTRICTrestrict quant_weights) { |
| 5071 | |
| 5072 | const int gindex = iq2_data_index(GGML_TYPE_IQ2_S); |
| 5073 | |
| 5074 | const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; |
| 5075 | const int * kmap_q2xs = iq2_data[gindex].map; |
| 5076 | const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; |
| 5077 | |
| 5078 | GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?")if (!(kmap_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5078, "GGML_ASSERT(%s) failed", "kmap_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 5079 | GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?")if (!(kgrid_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5079, "GGML_ASSERT(%s) failed", "kgrid_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 5080 | GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?")if (!(kneighbors_q2xs && "forgot to call ggml_quantize_init()?" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5080, "GGML_ASSERT(%s) failed", "kneighbors_q2xs && \"forgot to call ggml_quantize_init()?\"" ); |
| 5081 | GGML_ASSERT(n%QK_K == 0)if (!(n%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5081, "GGML_ASSERT(%s) failed", "n%QK_K == 0"); |
| 5082 | |
| 5083 | const int kMaxQ = 3; |
| 5084 | |
| 5085 | const int64_t nbl = n/QK_K256; |
| 5086 | |
| 5087 | block_iq2_s * y = vy; |
| 5088 | |
| 5089 | float scales[QK_K256/16]; |
| 5090 | float weight[16]; |
| 5091 | float xval[16]; |
| 5092 | int8_t L[16]; |
| 5093 | int8_t Laux[16]; |
| 5094 | float waux[16]; |
| 5095 | bool_Bool is_on_grid[2]; |
| 5096 | bool_Bool is_on_grid_aux[2]; |
| 5097 | uint8_t block_signs[2]; |
| 5098 | |
| 5099 | for (int ibl = 0; ibl < nbl; ++ibl) { |
| 5100 | |
| 5101 | memset(&y[ibl], 0, sizeof(block_iq2_s)); |
| 5102 | y[ibl].d = GGML_FP32_TO_FP16(0.f)ggml_compute_fp32_to_fp16(0.f); |
| 5103 | |
| 5104 | float max_scale = 0; |
| 5105 | |
| 5106 | const float * xbl = x + QK_K256*ibl; |
| 5107 | float sumx2 = 0; |
| 5108 | for (int i = 0; i < QK_K256; ++i) sumx2 += xbl[i]*xbl[i]; |
| 5109 | float sigma2 = 2*sumx2/QK_K256; |
| 5110 | |
| 5111 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 5112 | const float * xb = xbl + 16*ib; |
| 5113 | if (quant_weights) { |
| 5114 | const float * qw = quant_weights + QK_K256*ibl + 16*ib; |
| 5115 | for (int i = 0; i < 16; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); |
| 5116 | } else { |
| 5117 | for (int i = 0; i < 16; ++i) weight[i] = 0.25f*sigma2 + xb[i]*xb[i]; |
| 5118 | } |
| 5119 | for (int i = 0; i < 16; ++i) waux[i] = sqrtf(weight[i]); |
| 5120 | for (int k = 0; k < 2; ++k) { |
| 5121 | uint8_t s = 0; |
| 5122 | for (int i = 0; i < 8; ++i) { |
| 5123 | if (xb[8*k + i] >= 0) xval[8*k + i] = xb[8*k + i]; |
| 5124 | else { |
| 5125 | xval[8*k + i] = -xb[8*k + i]; s |= (1 << i); |
| 5126 | } |
| 5127 | } |
| 5128 | block_signs[k] = s; |
| 5129 | } |
| 5130 | float max = xval[0]; |
| 5131 | for (int i = 1; i < 16; ++i) max = MAX(max, xval[i])((max) > (xval[i]) ? (max) : (xval[i])); |
| 5132 | memset(L, 0, 16); |
| 5133 | if (max < GROUP_MAX_EPS_IQ2_S1e-8f) { |
| 5134 | scales[ib] = 0; |
| 5135 | continue; |
| 5136 | } |
| 5137 | float best = 0; |
| 5138 | float scale = max/(2*kMaxQ-1); |
| 5139 | is_on_grid[0] = is_on_grid[1] = true1; |
| 5140 | for (int is = -9; is <= 9; ++is) { |
| 5141 | float id = (2*kMaxQ-1+is*0.1f)/max; |
| 5142 | float this_scale = 1/id; |
| 5143 | for (int k = 0; k < 2; ++k) { |
| 5144 | for (int i = 0; i < 8; ++i) { |
| 5145 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 5146 | Laux[8*k+i] = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 5147 | } |
| 5148 | uint16_t u = 0; |
| 5149 | for (int i = 0; i < 8; ++i) u |= (Laux[8*k+i] << 2*i); |
| 5150 | int grid_index = kmap_q2xs[u]; |
| 5151 | is_on_grid_aux[k] = true1; |
| 5152 | if (grid_index < 0) { |
| 5153 | is_on_grid_aux[k] = false0; |
| 5154 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 5155 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k); |
| 5156 | } |
| 5157 | } |
| 5158 | float sumqx = 0, sumq2 = 0; |
| 5159 | for (int i = 0; i < 16; ++i) { |
| 5160 | float w = weight[i]; |
| 5161 | float q = 2*Laux[i] + 1; |
| 5162 | sumqx += w*xval[i]*q; |
| 5163 | sumq2 += w*q*q; |
| 5164 | } |
| 5165 | if (sumq2 > 0 && sumqx*sumqx > best*sumq2) { |
| 5166 | scale = sumqx/sumq2; best = scale*sumqx; |
| 5167 | for (int i = 0; i < 16; ++i) L[i] = Laux[i]; |
| 5168 | for (int k = 0; k < 2; ++k) is_on_grid[k] = is_on_grid_aux[k]; |
| 5169 | } |
| 5170 | } |
| 5171 | int n_not_ongrid = 0; |
| 5172 | for (int k = 0; k < 2; ++k) if (!is_on_grid[k]) ++n_not_ongrid; |
| 5173 | if (n_not_ongrid > 0 && scale > 0) { |
| 5174 | float id = 1/scale; |
| 5175 | for (int k = 0; k < 2; ++k) { |
| 5176 | if (is_on_grid[k]) continue; |
| 5177 | uint16_t u = 0; |
| 5178 | for (int i = 0; i < 8; ++i) { |
| 5179 | int l = nearest_int(0.5f*(id*xval[8*k+i]-1)); |
| 5180 | l = MAX(0, MIN(kMaxQ-1, l))((0) > (((kMaxQ-1) < (l) ? (kMaxQ-1) : (l))) ? (0) : (( (kMaxQ-1) < (l) ? (kMaxQ-1) : (l)))); |
| 5181 | u |= (l << 2*i); |
| 5182 | L[8*k + i] = l; |
| 5183 | } |
| 5184 | int grid_index = kmap_q2xs[u]; |
| 5185 | if (grid_index < 0) { |
| 5186 | const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; |
| 5187 | grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k); |
| 5188 | } |
| 5189 | } |
| 5190 | float sumqx = 0, sumq2 = 0; |
| 5191 | for (int i = 0; i < 16; ++i) { |
| 5192 | float w = weight[i]; |
| 5193 | float q = 2*L[i] + 1; |
| 5194 | sumqx += w*xval[i]*q; |
| 5195 | sumq2 += w*q*q; |
| 5196 | } |
| 5197 | if (sumq2 > 0) scale = sumqx/sumq2; |
| 5198 | } |
| 5199 | if (scale < 0) { |
| 5200 | scale = -scale; |
| 5201 | for (int k = 0; k < 2; ++k) block_signs[k] = ~block_signs[k]; |
| 5202 | } |
| 5203 | for (int k = 0; k < 2; ++k) { |
| 5204 | uint16_t u = 0; |
| 5205 | for (int i = 0; i < 8; ++i) u |= (L[8*k+i] << 2*i); |
| 5206 | int grid_index = kmap_q2xs[u]; |
| 5207 | if (grid_index < 0) { |
| 5208 | printf("Oops: found point %u not on grid:", u); |
| 5209 | for (int i = 0; i < 8; ++i) printf(" %d", L[8*k+i]); |
| 5210 | printf("\n"); |
| 5211 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5211, "fatal error"); |
| 5212 | } |
| 5213 | const int i8 = 2*ib + k; |
| 5214 | y[ibl].qs[i8] = grid_index & 255; |
| 5215 | y[ibl].qh[i8/4] |= ((grid_index >> 8) << 2*(i8%4)); |
| 5216 | y[ibl].qs[QK_K256/8 + i8] = block_signs[k]; |
| 5217 | } |
| 5218 | GGML_ASSERT(scale >= 0)if (!(scale >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5218, "GGML_ASSERT(%s) failed", "scale >= 0"); |
| 5219 | scales[ib] = scale; |
| 5220 | max_scale = MAX(max_scale, scale)((max_scale) > (scale) ? (max_scale) : (scale)); |
| 5221 | } |
| 5222 | |
| 5223 | if (!max_scale) { |
| 5224 | continue; |
| 5225 | } |
| 5226 | |
| 5227 | float d = max_scale/31; |
| 5228 | y[ibl].d = GGML_FP32_TO_FP16(d * 0.9875f)ggml_compute_fp32_to_fp16(d * 0.9875f); |
| 5229 | float id = 1/d; |
| 5230 | for (int ib = 0; ib < QK_K256/16; ++ib) { |
| 5231 | int l = nearest_int(0.5f*(id*scales[ib]-1)); |
| 5232 | l = MAX(0, MIN(15, l))((0) > (((15) < (l) ? (15) : (l))) ? (0) : (((15) < ( l) ? (15) : (l)))); |
| 5233 | if (ib%2 == 0) y[ibl].scales[ib/2] = l; |
| 5234 | else y[ibl].scales[ib/2] |= (l << 4); |
| 5235 | } |
| 5236 | } |
| 5237 | } |
| 5238 | |
| 5239 | size_t quantize_iq2_s(const float * GGML_RESTRICTrestrict src, void * GGML_RESTRICTrestrict dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { |
| 5240 | GGML_ASSERT(n_per_row%QK_K == 0)if (!(n_per_row%256 == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5240, "GGML_ASSERT(%s) failed", "n_per_row%QK_K == 0"); |
| 5241 | int64_t nblock = n_per_row/QK_K256; |
| 5242 | char * qrow = (char *)dst; |
| 5243 | for (int64_t row = 0; row < nrow; ++row) { |
| 5244 | quantize_row_iq2_s_impl(src, qrow, n_per_row, quant_weights); |
| 5245 | src += n_per_row; |
| 5246 | qrow += nblock*sizeof(block_iq2_s); |
| 5247 | } |
| 5248 | return nrow * nblock * sizeof(block_iq2_s); |
| 5249 | } |
| 5250 | |
| 5251 | void quantize_row_iq2_s_ref(const float * GGML_RESTRICTrestrict x, block_iq2_s * GGML_RESTRICTrestrict y, int64_t k) { |
| 5252 | assert(k % QK_K == 0)((void) sizeof (__assert_single_arg (k % 256 == 0)), __extension__ ({ if (k % 256 == 0) ; else __assert_fail ("k % QK_K == 0", "/root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-quants.c" , 5252, __extension__ __PRETTY_FUNCTION__); })); |
| 5253 | quantize_iq2_s(x, y, 1, k, NULL((void*)0)); |
| 5254 | } |
| 5255 | |
| 5256 | // =============================== data validation |
| 5257 | |
| 5258 | static bool_Bool validate_float(float f, size_t i) { |
| 5259 | if (isinf(f)__builtin_isinf_sign (f)) { |
| 5260 | fprintf(stderrstderr, "ggml_validate_row_data: found inf value at block %zu\n", i); |
| 5261 | return false0; |
| 5262 | } |
| 5263 | |
| 5264 | if (isnan(f)__builtin_isnan (f)) { |
| 5265 | fprintf(stderrstderr, "ggml_validate_row_data: found nan value at block %zu\n", i); |
| 5266 | return false0; |
| 5267 | } |
| 5268 | |
| 5269 | return true1; |
| 5270 | } |
| 5271 | |
| 5272 | static bool_Bool isinf_fp16(ggml_fp16_t f) { |
| 5273 | return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) == 0; |
| 5274 | } |
| 5275 | |
| 5276 | static bool_Bool isnan_fp16(ggml_fp16_t f) { |
| 5277 | return (f & 0x7c00) == 0x7c00 && (f & 0x03ff) != 0; |
| 5278 | } |
| 5279 | |
| 5280 | static bool_Bool validate_fp16(ggml_fp16_t f, size_t i) { |
| 5281 | if (isinf_fp16(f)) { |
| 5282 | fprintf(stderrstderr, "ggml_validate_row_data: found inf value at block %zu\n", i); |
| 5283 | return false0; |
| 5284 | } |
| 5285 | |
| 5286 | if (isnan_fp16(f)) { |
| 5287 | fprintf(stderrstderr, "ggml_validate_row_data: found nan value at block %zu\n", i); |
| 5288 | return false0; |
| 5289 | } |
| 5290 | |
| 5291 | return true1; |
| 5292 | } |
| 5293 | |
| 5294 | static bool_Bool validate_e_e8m0(uint8_t e, size_t i) { |
| 5295 | if (e == 0xff) { |
| 5296 | fprintf(stderrstderr, "ggml_validate_row_data: found invalid e value %d at block %zu\n", e, i); |
| 5297 | return false0; |
| 5298 | } |
| 5299 | |
| 5300 | return true1; |
| 5301 | } |
| 5302 | |
| 5303 | #define VALIDATE_ROW_DATA_D_F16_IMPL(type, data, nb)const type * q = (const type *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } } \ |
| 5304 | const type * q = (const type *) (data); \ |
| 5305 | for (size_t i = 0; i < (nb); ++i) { \ |
| 5306 | if (!validate_fp16(q[i].d, i)) { \ |
| 5307 | return false0; \ |
| 5308 | } \ |
| 5309 | } |
| 5310 | |
| 5311 | #define VALIDATE_ROW_DATA_DM_F16_IMPL(type, data, nb, d, m)const type * q = (const type *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || !validate_fp16 (q[i].m, i)) { return 0; } } \ |
| 5312 | const type * q = (const type *) (data); \ |
| 5313 | for (size_t i = 0; i < (nb); ++i) { \ |
| 5314 | if (!validate_fp16(q[i].d, i) || !validate_fp16(q[i].m, i)) { \ |
| 5315 | return false0; \ |
| 5316 | } \ |
| 5317 | } |
| 5318 | |
| 5319 | #define VALIDATE_ROW_DATA_E_E8M0_IMPL(type, data, nb)const type * q = (const type *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_e_e8m0(q[i].e, i)) { return 0; } } \ |
| 5320 | const type * q = (const type *) (data); \ |
| 5321 | for (size_t i = 0; i < (nb); ++i) { \ |
| 5322 | if (!validate_e_e8m0(q[i].e, i)) { \ |
| 5323 | return false0; \ |
| 5324 | } \ |
| 5325 | } |
| 5326 | |
| 5327 | #define VALIDATE_ROW_DATA_DVEC_F16_IMPL(type, data, nb, nr)const type * q = (const type *) (data); for (size_t i = 0; i < (nb); ++i) { for (size_t j = 0; j < (nr); ++j) { if (!validate_fp16 (q[i].d[j], i)) { return 0; } } } \ |
| 5328 | const type * q = (const type *) (data); \ |
| 5329 | for (size_t i = 0; i < (nb); ++i) { \ |
| 5330 | for (size_t j = 0; j < (nr); ++j) { \ |
| 5331 | if (!validate_fp16(q[i].d[j], i)) { \ |
| 5332 | return false0; \ |
| 5333 | } \ |
| 5334 | } \ |
| 5335 | } |
| 5336 | |
| 5337 | bool_Bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes) { |
| 5338 | if (type < 0 || type >= GGML_TYPE_COUNT) { |
| 5339 | fprintf(stderrstderr, "%s: invalid type %d\n", __func__, type); |
| 5340 | return false0; |
| 5341 | } |
| 5342 | |
| 5343 | if (nbytes % ggml_type_size(type) != 0) { |
| 5344 | fprintf(stderrstderr, "%s: invalid size %zu for type %s (type size = %zu)\n", __func__, nbytes, ggml_type_name(type), ggml_type_size(type)); |
| 5345 | return false0; |
| 5346 | } |
| 5347 | |
| 5348 | const size_t nb = nbytes/ggml_type_size(type); |
| 5349 | |
| 5350 | switch (type) { |
| 5351 | case GGML_TYPE_BF16: |
| 5352 | { |
| 5353 | int nans = 0; |
| 5354 | int infs = 0; |
| 5355 | const unsigned short * f = (const unsigned short *) data; |
| 5356 | for (size_t i = 0; i < nb; ++i) { |
| 5357 | nans += (f[i] & 0x7fff) > 0x7f80; |
| 5358 | infs += (f[i] & 0x7fff) == 0x7f80; |
| 5359 | } |
| 5360 | if (nans) { |
| 5361 | fprintf(stderrstderr, "%s: found %d NaNs in row of %zu BF16 values\n", __func__, nans, nb); |
| 5362 | return false0; |
| 5363 | } |
| 5364 | if (infs) { |
| 5365 | fprintf(stderrstderr, "%s: found %d infinities in row of %zu BF16 values\n", __func__, infs, nb); |
| 5366 | return false0; |
| 5367 | } |
| 5368 | } break; |
| 5369 | case GGML_TYPE_F16: |
| 5370 | { |
| 5371 | const ggml_fp16_t * f = (const ggml_fp16_t *) data; |
| 5372 | size_t i = 0; |
| 5373 | #if defined(__AVX2__1) |
| 5374 | for (; i + 15 < nb; i += 16) { |
| 5375 | __m256i v = _mm256_loadu_si256((const __m256i *)(f + i)); |
| 5376 | __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi16(0x7c00)); |
| 5377 | __m256i cmp = _mm256_cmpeq_epi16(vexp, _mm256_set1_epi16(0x7c00)); |
| 5378 | int mask = _mm256_movemask_epi8(cmp); |
| 5379 | if (mask) { |
| 5380 | for (size_t j = 0; j < 16; ++j) { |
| 5381 | if (!validate_fp16(f[i + j], i + j)) { |
| 5382 | return false0; |
| 5383 | } |
| 5384 | } |
| 5385 | GGML_UNREACHABLE()do { fprintf(stderr, "statement should be unreachable\n"); abort (); } while(0); |
| 5386 | } |
| 5387 | } |
| 5388 | #elif defined(__ARM_NEON) |
| 5389 | for (; i + 7 < nb; i += 8) { |
| 5390 | uint16x8_t v = vld1q_u16(f + i); |
| 5391 | uint16x8_t vexp = vandq_u16(v, vdupq_n_u16(0x7c00)); |
| 5392 | uint16x8_t cmp = vceqq_u16(vexp, vdupq_n_u16(0x7c00)); |
| 5393 | uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(cmp, 4)), 0); |
| 5394 | if (mask) { |
| 5395 | for (size_t j = 0; j < 8; ++j) { |
| 5396 | if (!validate_fp16(f[i + j], i + j)) { |
| 5397 | return false0; |
| 5398 | } |
| 5399 | } |
| 5400 | GGML_UNREACHABLE()do { fprintf(stderr, "statement should be unreachable\n"); abort (); } while(0); |
| 5401 | } |
| 5402 | } |
| 5403 | #endif |
| 5404 | for (; i < nb; ++i) { |
| 5405 | if (!validate_fp16(f[i], i)) { |
| 5406 | return false0; |
| 5407 | } |
| 5408 | } |
| 5409 | } break; |
| 5410 | case GGML_TYPE_F32: |
| 5411 | { |
| 5412 | const float * f = (const float *) data; |
| 5413 | size_t i = 0; |
| 5414 | #if defined(__AVX2__1) |
| 5415 | for (; i + 7 < nb; i += 8) { |
| 5416 | __m256i v = _mm256_loadu_si256((const __m256i *)(f + i)); |
| 5417 | __m256i vexp = _mm256_and_si256(v, _mm256_set1_epi32(0x7f800000)); |
| 5418 | __m256i cmp = _mm256_cmpeq_epi32(vexp, _mm256_set1_epi32(0x7f800000)); |
| 5419 | int mask = _mm256_movemask_epi8(cmp); |
| 5420 | if (mask) { |
| 5421 | for (size_t j = 0; j < 8; ++j) { |
| 5422 | if (!validate_float(f[i + j], i + j)) { |
| 5423 | return false0; |
| 5424 | } |
| 5425 | } |
| 5426 | GGML_UNREACHABLE()do { fprintf(stderr, "statement should be unreachable\n"); abort (); } while(0); |
| 5427 | } |
| 5428 | } |
| 5429 | #elif defined(__ARM_NEON) |
| 5430 | for (; i + 3 < nb; i += 4) { |
| 5431 | uint32x4_t v = vld1q_u32((const uint32_t *)f + i); |
| 5432 | uint32x4_t vexp = vandq_u32(v, vdupq_n_u32(0x7f800000)); |
| 5433 | uint32x4_t cmp = vceqq_u32(vexp, vdupq_n_u32(0x7f800000)); |
| 5434 | uint64_t mask = vget_lane_u64(vreinterpret_u64_u16(vshrn_n_u32(cmp, 8)), 0); |
| 5435 | if (mask) { |
| 5436 | for (size_t j = 0; j < 4; ++j) { |
| 5437 | if (!validate_float(f[i + j], i + j)) { |
| 5438 | return false0; |
| 5439 | } |
| 5440 | } |
| 5441 | GGML_UNREACHABLE()do { fprintf(stderr, "statement should be unreachable\n"); abort (); } while(0); |
| 5442 | } |
| 5443 | } |
| 5444 | #endif |
| 5445 | for (; i < nb; ++i) { |
| 5446 | if (!validate_float(f[i], i)) { |
| 5447 | return false0; |
| 5448 | } |
| 5449 | } |
| 5450 | } break; |
| 5451 | case GGML_TYPE_F64: |
| 5452 | { |
| 5453 | const double * f = (const double *) data; |
| 5454 | for (size_t i = 0; i < nb; ++i) { |
| 5455 | if (!validate_float(f[i], i)) { |
| 5456 | return false0; |
| 5457 | } |
| 5458 | } |
| 5459 | } break; |
| 5460 | case GGML_TYPE_Q1_0: |
| 5461 | { |
| 5462 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q1_0, data, nb)const block_q1_0 * q = (const block_q1_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5463 | } break; |
| 5464 | case GGML_TYPE_Q4_0: |
| 5465 | { |
| 5466 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_0, data, nb)const block_q4_0 * q = (const block_q4_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5467 | } break; |
| 5468 | case GGML_TYPE_Q4_1: |
| 5469 | { |
| 5470 | VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_1, data, nb, d, m)const block_q4_1 * q = (const block_q4_1 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || ! validate_fp16(q[i].m, i)) { return 0; } }; |
| 5471 | } break; |
| 5472 | case GGML_TYPE_Q5_0: |
| 5473 | { |
| 5474 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q5_0, data, nb)const block_q5_0 * q = (const block_q5_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5475 | } break; |
| 5476 | case GGML_TYPE_Q5_1: |
| 5477 | { |
| 5478 | VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_1, data, nb, d, m)const block_q5_1 * q = (const block_q5_1 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || ! validate_fp16(q[i].m, i)) { return 0; } }; |
| 5479 | } break; |
| 5480 | case GGML_TYPE_Q8_0: |
| 5481 | { |
| 5482 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q8_0, data, nb)const block_q8_0 * q = (const block_q8_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5483 | } break; |
| 5484 | case GGML_TYPE_MXFP4: |
| 5485 | { |
| 5486 | VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp4, data, nb)const block_mxfp4 * q = (const block_mxfp4 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_e_e8m0(q[i].e, i)) { return 0; } }; |
| 5487 | } break; |
| 5488 | case GGML_TYPE_NVFP4: |
| 5489 | { |
| 5490 | // UE4M3 scales are uint8_t — all byte values are valid |
| 5491 | GGML_UNUSED(data)(void)(data); |
| 5492 | GGML_UNUSED(nb)(void)(nb); |
| 5493 | } break; |
| 5494 | case GGML_TYPE_Q2_K: |
| 5495 | { |
| 5496 | VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_K, data, nb, d, dmin)const block_q2_K * q = (const block_q2_K *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || ! validate_fp16(q[i].dmin, i)) { return 0; } }; |
| 5497 | } break; |
| 5498 | case GGML_TYPE_Q3_K: |
| 5499 | { |
| 5500 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_K, data, nb)const block_q3_K * q = (const block_q3_K *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5501 | } break; |
| 5502 | case GGML_TYPE_Q4_K: |
| 5503 | { |
| 5504 | VALIDATE_ROW_DATA_DM_F16_IMPL(block_q4_K, data, nb, d, dmin)const block_q4_K * q = (const block_q4_K *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || ! validate_fp16(q[i].dmin, i)) { return 0; } }; |
| 5505 | } break; |
| 5506 | case GGML_TYPE_Q5_K: |
| 5507 | { |
| 5508 | VALIDATE_ROW_DATA_DM_F16_IMPL(block_q5_K, data, nb, d, dmin)const block_q5_K * q = (const block_q5_K *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i) || ! validate_fp16(q[i].dmin, i)) { return 0; } }; |
| 5509 | } break; |
| 5510 | case GGML_TYPE_Q6_K: |
| 5511 | { |
| 5512 | VALIDATE_ROW_DATA_D_F16_IMPL(block_q6_K, data, nb)const block_q6_K * q = (const block_q6_K *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5513 | } break; |
| 5514 | case GGML_TYPE_Q8_K: |
| 5515 | { |
| 5516 | const block_q8_K * q = (const block_q8_K *) data; |
| 5517 | for (size_t i = 0; i < nb; ++i) { |
| 5518 | if (!validate_float(q[i].d, i)) { |
| 5519 | return false0; |
| 5520 | } |
| 5521 | } |
| 5522 | } break; |
| 5523 | case GGML_TYPE_TQ1_0: |
| 5524 | { |
| 5525 | VALIDATE_ROW_DATA_D_F16_IMPL(block_tq1_0, data, nb)const block_tq1_0 * q = (const block_tq1_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5526 | } break; |
| 5527 | case GGML_TYPE_TQ2_0: |
| 5528 | { |
| 5529 | VALIDATE_ROW_DATA_D_F16_IMPL(block_tq2_0, data, nb)const block_tq2_0 * q = (const block_tq2_0 *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5530 | } break; |
| 5531 | case GGML_TYPE_IQ1_S: |
| 5532 | { |
| 5533 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb)const block_iq1_s * q = (const block_iq1_s *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5534 | } break; |
| 5535 | case GGML_TYPE_IQ1_M: |
| 5536 | { |
| 5537 | const block_iq1_m * q = (const block_iq1_m *) data; |
| 5538 | for (size_t i = 0; i < nb; ++i) { |
| 5539 | iq1m_scale_t scale; |
| 5540 | const uint16_t * sc = (const uint16_t *)q[i].scales; |
| 5541 | scale.u16 = (sc[0] >> 12) | ((sc[1] >> 8) & 0x00f0) | ((sc[2] >> 4) & 0x0f00) | (sc[3] & 0xf000); |
| 5542 | if (!validate_fp16(scale.f16, i)) { |
| 5543 | return false0; |
| 5544 | } |
| 5545 | } |
| 5546 | } break; |
| 5547 | case GGML_TYPE_IQ2_XXS: |
| 5548 | { |
| 5549 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xxs, data, nb)const block_iq2_xxs * q = (const block_iq2_xxs *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d , i)) { return 0; } }; |
| 5550 | } break; |
| 5551 | case GGML_TYPE_IQ2_XS: |
| 5552 | { |
| 5553 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_xs, data, nb)const block_iq2_xs * q = (const block_iq2_xs *) (data); for ( size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i )) { return 0; } }; |
| 5554 | } break; |
| 5555 | case GGML_TYPE_IQ2_S: |
| 5556 | { |
| 5557 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_s, data, nb)const block_iq2_s * q = (const block_iq2_s *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5558 | } break; |
| 5559 | case GGML_TYPE_IQ3_XXS: |
| 5560 | { |
| 5561 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_xxs, data, nb)const block_iq3_xxs * q = (const block_iq3_xxs *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d , i)) { return 0; } }; |
| 5562 | } break; |
| 5563 | |
| 5564 | case GGML_TYPE_IQ3_S: |
| 5565 | { |
| 5566 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_s, data, nb)const block_iq3_s * q = (const block_iq3_s *) (data); for (size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i)) { return 0; } }; |
| 5567 | } break; |
| 5568 | case GGML_TYPE_IQ4_XS: |
| 5569 | { |
| 5570 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_xs, data, nb)const block_iq4_xs * q = (const block_iq4_xs *) (data); for ( size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i )) { return 0; } }; |
| 5571 | } break; |
| 5572 | case GGML_TYPE_IQ4_NL: |
| 5573 | { |
| 5574 | VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb)const block_iq4_nl * q = (const block_iq4_nl *) (data); for ( size_t i = 0; i < (nb); ++i) { if (!validate_fp16(q[i].d, i )) { return 0; } }; |
| 5575 | } break; |
| 5576 | |
| 5577 | case GGML_TYPE_I8: |
| 5578 | case GGML_TYPE_I16: |
| 5579 | case GGML_TYPE_I32: |
| 5580 | case GGML_TYPE_I64: |
| 5581 | // nothing to validate |
| 5582 | break; |
| 5583 | default: |
| 5584 | { |
| 5585 | fprintf(stderrstderr, "%s: invalid type %d\n", __func__, type); |
| 5586 | return false0; |
| 5587 | } |
| 5588 | } |
| 5589 | |
| 5590 | return true1; |
| 5591 | } |