| File: | root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp |
| Warning: | line 1310, column 12 Value stored to 'gf' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | #include "llama-context.h" |
| 2 | |
| 3 | #include "ggml.h" |
| 4 | #include "llama-arch.h" |
| 5 | #include "llama-graph.h" |
| 6 | #include "llama-impl.h" |
| 7 | #include "llama-batch.h" |
| 8 | #include "llama-io.h" |
| 9 | #include "llama-memory.h" |
| 10 | #include "llama-mmap.h" |
| 11 | #include "llama-model.h" |
| 12 | #include "llama-ext.h" |
| 13 | #include "llama.h" |
| 14 | |
| 15 | #include <cinttypes> |
| 16 | #include <cmath> |
| 17 | #include <cstring> |
| 18 | #include <limits> |
| 19 | #include <stdexcept> |
| 20 | |
| 21 | #include "moz-overrides.h" |
| 22 | |
| 23 | // |
| 24 | // llama_context |
| 25 | // |
| 26 | |
| 27 | static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { |
| 28 | switch (ctx_type) { |
| 29 | case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT; |
| 30 | case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP; |
| 31 | } |
| 32 | throwabort_with_suppression(); if (false) std::runtime_error("Unsupported ctx type"); |
| 33 | } |
| 34 | |
| 35 | llama_context::llama_context( |
| 36 | const llama_model & model, |
| 37 | llama_context_params params) : |
| 38 | model(model), |
| 39 | cvec(std::make_unique<llama_adapter_cvec>()), |
| 40 | loras(std::make_unique<llama_adapter_loras>()), |
| 41 | balloc(std::make_unique<llama_batch_allocr>(model.hparams.n_pos_per_embd())) { |
| 42 | // TODO warning when creating llama_context with awkward ctx size that is not a power of 2, |
| 43 | // may need to be backend-dependent |
| 44 | LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: constructing llama_context\n" , __func__); |
| 45 | |
| 46 | t_start_us = model.t_start_us; |
| 47 | t_load_us = model.t_load_us; |
| 48 | |
| 49 | const auto & hparams = model.hparams; |
| 50 | |
| 51 | cparams.n_seq_max = std::max(1u, params.n_seq_max); |
| 52 | if (cparams.n_seq_max > LLAMA_MAX_SEQ256) { |
| 53 | throwabort_with_suppression(); if (false) std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ256)); |
| 54 | } |
| 55 | |
| 56 | cparams.n_rs_seq = params.n_rs_seq; |
| 57 | if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) { |
| 58 | LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n" , __func__, cparams.n_rs_seq) |
| 59 | __func__, cparams.n_rs_seq)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n" , __func__, cparams.n_rs_seq); |
| 60 | cparams.n_rs_seq = 0; |
| 61 | } |
| 62 | |
| 63 | cparams.n_threads = params.n_threads; |
| 64 | cparams.n_threads_batch = params.n_threads_batch; |
| 65 | cparams.yarn_ext_factor = params.yarn_ext_factor >= 0.0f ? params.yarn_ext_factor : hparams.yarn_ext_factor; |
| 66 | cparams.yarn_attn_factor = params.yarn_attn_factor >= 0.0f ? params.yarn_attn_factor : hparams.yarn_attn_factor; |
| 67 | cparams.yarn_beta_fast = params.yarn_beta_fast >= 0.0f ? params.yarn_beta_fast : hparams.yarn_beta_fast; |
| 68 | cparams.yarn_beta_slow = params.yarn_beta_slow >= 0.0f ? params.yarn_beta_slow : hparams.yarn_beta_slow; |
| 69 | cparams.embeddings = params.embeddings; |
| 70 | cparams.embeddings_nextn = false; |
| 71 | cparams.embeddings_nextn_masked = false; |
| 72 | cparams.offload_kqv = params.offload_kqv; |
| 73 | cparams.no_perf = params.no_perf; |
| 74 | cparams.warmup = false; |
| 75 | |
| 76 | cparams.embeddings_layer_inp.resize(hparams.n_layer(), false); |
| 77 | embd_layer_inp.resize(hparams.n_layer()); |
| 78 | |
| 79 | cparams.ctx_type = params.ctx_type; |
| 80 | cparams.pooling_type = params.pooling_type; |
| 81 | |
| 82 | cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx; |
| 83 | cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base; |
| 84 | cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale; |
| 85 | |
| 86 | cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx : |
| 87 | hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn : |
| 88 | hparams.n_ctx_train; |
| 89 | |
| 90 | cparams.cb_eval = params.cb_eval; |
| 91 | cparams.cb_eval_user_data = params.cb_eval_user_data; |
| 92 | |
| 93 | cparams.ctx_other = nullptr; |
| 94 | |
| 95 | // TODO: more generic |
| 96 | if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) { |
| 97 | if (params.ctx_other == nullptr) { |
| 98 | // TODO: change from runtime_error to llama_exception to avoid printing error message |
| 99 | throwabort_with_suppression(); if (false) std::runtime_error("Gemma4Assistant requires ctx_other to be set (this warning is normal during memory fitting)"); |
| 100 | } |
| 101 | |
| 102 | cparams.ctx_other = params.ctx_other; |
| 103 | } |
| 104 | |
| 105 | if (model.arch == LLM_ARCH_EAGLE3) { |
| 106 | if (model.tok_embd == nullptr || model.output == nullptr) { |
| 107 | if (params.ctx_other == nullptr) { |
| 108 | throwabort_with_suppression(); if (false) std::runtime_error("EAGLE3 requires ctx_other to be set (this warning is normal during memory fitting)"); |
| 109 | } |
| 110 | cparams.ctx_other = params.ctx_other; |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // Initialize backend samplers here so they are part of the sampling graph |
| 115 | // before the reserve passes run later in this function. This avoids a later |
| 116 | // re-reserve when graph nodes change. |
| 117 | if (params.samplers != nullptr && params.n_samplers > 0) { |
| 118 | for (size_t i = 0; i < params.n_samplers; ++i) { |
| 119 | const auto & config = params.samplers[i]; |
| 120 | |
| 121 | if (llama_sampler_chain_get(config.sampler, -1) == nullptr) { |
| 122 | throwabort_with_suppression(); if (false) std::runtime_error("the backend samplers must be of type llama_sampler_chain"); |
| 123 | } |
| 124 | |
| 125 | if (set_sampler(config.seq_id, config.sampler)) { |
| 126 | const int n_samplers = llama_sampler_chain_n(config.sampler); |
| 127 | |
| 128 | LLAMA_LOG_INFO("%s: setting backend sampler for seq_id %d (n = %d)\n", __func__, config.seq_id, n_samplers)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: setting backend sampler for seq_id %d (n = %d)\n" , __func__, config.seq_id, n_samplers); |
| 129 | } |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | auto rope_scaling_type = params.rope_scaling_type; |
| 134 | if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) { |
| 135 | rope_scaling_type = hparams.rope_scaling_type_train; |
| 136 | } |
| 137 | |
| 138 | if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) { |
| 139 | cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none |
| 140 | } |
| 141 | |
| 142 | if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set' |
| 143 | cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f; |
| 144 | } |
| 145 | |
| 146 | if (cparams.yarn_ext_factor != 0) { |
| 147 | static auto get_mscale = [](float scale, float mscale) { |
| 148 | return scale <= 1.0f ? 1.0f : (0.1f * mscale * logf(scale) + 1.0f); |
| 149 | }; |
| 150 | |
| 151 | const float factor = 1.0f / cparams.rope_freq_scale; |
| 152 | |
| 153 | // ref: https://github.com/huggingface/transformers/blob/6d00f6b0a5679c36510f203e4226e36f517c3032/src/transformers/modeling_rope_utils.py#L336-L348 |
| 154 | if (hparams.rope_yarn_log_mul != 0.0f) { |
| 155 | // note: here we assume `mscale == 1.0f` |
| 156 | // TODO: start reading the actual value of mscale and handle the case where it is not 1.0f |
| 157 | float mscale = 1.0f; |
| 158 | const float mscale_all_dims = hparams.rope_yarn_log_mul; |
| 159 | |
| 160 | // [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] |
| 161 | // special-case DEEPSEEK v2: |
| 162 | // https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/main/config.json#L42-L43 |
| 163 | if (model.arch == LLM_ARCH_DEEPSEEK2 && mscale_all_dims != 1.0f) { |
| 164 | mscale = mscale_all_dims; |
| 165 | } |
| 166 | |
| 167 | cparams.yarn_attn_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dims); |
| 168 | |
| 169 | LLAMA_LOG_WARN("%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n" , __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims ) |
| 170 | __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n" , __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims ); |
| 171 | } else { |
| 172 | cparams.yarn_attn_factor = get_mscale(factor, 1.0f); |
| 173 | } |
| 174 | |
| 175 | // when YARN is applied with yarn_ext_factor != 0.0f, we need to cancel this factor: |
| 176 | // https://github.com/ggml-org/llama.cpp/blob/a81a569577cc38b32558958b048228150be63eae/ggml/src/ggml-cpu/ops.cpp#L5541-L5544 |
| 177 | // |
| 178 | // ref: https://github.com/ggml-org/llama.cpp/discussions/7416 |
| 179 | // https://github.com/ggml-org/llama.cpp/pull/17945 |
| 180 | cparams.yarn_attn_factor *= 1.0f / (1.0f + 0.1f * logf(factor)); |
| 181 | } |
| 182 | |
| 183 | cparams.yarn_attn_factor *= hparams.rope_attn_factor; |
| 184 | |
| 185 | if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { |
| 186 | if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) { |
| 187 | cparams.pooling_type = LLAMA_POOLING_TYPE_NONE; |
| 188 | } else { |
| 189 | cparams.pooling_type = hparams.pooling_type; |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) { |
| 194 | cparams.causal_attn = hparams.causal_attn; |
| 195 | } else { |
| 196 | cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL; |
| 197 | } |
| 198 | |
| 199 | cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED; |
| 200 | cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO; |
| 201 | |
| 202 | cparams.fused_gdn_ar = true; |
| 203 | cparams.fused_gdn_ch = true; |
| 204 | cparams.auto_fgdn = true; |
| 205 | |
| 206 | // with causal attention, the batch size is limited by the context size |
| 207 | cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch; |
| 208 | |
| 209 | cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch); |
| 210 | |
| 211 | cparams.n_outputs_max = params.n_outputs_max == 0 || llama_model_has_encoder(&model) ? cparams.n_batch : params.n_outputs_max; |
| 212 | |
| 213 | cparams.op_offload = params.op_offload; |
| 214 | cparams.kv_unified = params.kv_unified; |
| 215 | |
| 216 | // initialized later |
| 217 | cparams.pipeline_parallel = false; |
| 218 | |
| 219 | { |
| 220 | const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE"); |
| 221 | graph_reuse_disable = LLAMA_GRAPH_REUSE_DISABLE ? (atoi(LLAMA_GRAPH_REUSE_DISABLE) != 0) : graph_reuse_disable; |
| 222 | |
| 223 | if (graph_reuse_disable) { |
| 224 | LLAMA_LOG_WARN("%s: graph reuse disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: graph reuse disabled\n" , __func__); |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 |
| 229 | cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256)(((cparams.n_ctx) + (256) - 1) & ~((256) - 1)); |
| 230 | |
| 231 | if (cparams.kv_unified) { |
| 232 | cparams.n_ctx_seq = cparams.n_ctx; |
| 233 | } else { |
| 234 | cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max; |
| 235 | cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256)(((cparams.n_ctx_seq) + (256) - 1) & ~((256) - 1)); |
| 236 | |
| 237 | if (cparams.n_ctx_seq == 0) { |
| 238 | throwabort_with_suppression(); if (false) std::runtime_error("n_ctx_seq == 0"); |
| 239 | } |
| 240 | |
| 241 | if (cparams.n_ctx != cparams.n_ctx_seq * cparams.n_seq_max) { |
| 242 | cparams.n_ctx = cparams.n_ctx_seq * cparams.n_seq_max; |
| 243 | LLAMA_LOG_WARN("%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n", __func__, cparams.n_ctx)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n" , __func__, cparams.n_ctx); |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_seq_max = %u\n" , __func__, cparams.n_seq_max); |
| 248 | LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ctx = %u\n" , __func__, cparams.n_ctx); |
| 249 | LLAMA_LOG_INFO("%s: n_ctx_seq = %u\n", __func__, cparams.n_ctx_seq)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ctx_seq = %u\n" , __func__, cparams.n_ctx_seq); |
| 250 | LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_batch = %u\n" , __func__, cparams.n_batch); |
| 251 | LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ubatch = %u\n" , __func__, cparams.n_ubatch); |
| 252 | LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: causal_attn = %d\n" , __func__, cparams.causal_attn); |
| 253 | LLAMA_LOG_INFO("%s: flash_attn = %s\n", __func__, llama_flash_attn_type_name(params.flash_attn_type))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: flash_attn = %s\n" , __func__, llama_flash_attn_type_name(params.flash_attn_type )); |
| 254 | LLAMA_LOG_INFO("%s: kv_unified = %s\n", __func__, cparams.kv_unified ? "true" : "false")llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: kv_unified = %s\n" , __func__, cparams.kv_unified ? "true" : "false"); |
| 255 | LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: freq_base = %.1f\n" , __func__, cparams.rope_freq_base); |
| 256 | LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: freq_scale = %g\n" , __func__, cparams.rope_freq_scale); |
| 257 | LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_rs_seq = %u\n" , __func__, cparams.n_rs_seq); |
| 258 | LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_outputs_max = %u\n" , __func__, cparams.n_outputs_max); |
| 259 | |
| 260 | if (cparams.n_ctx_seq < hparams.n_ctx_train) { |
| 261 | LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train) |
| 262 | __func__, cparams.n_ctx_seq, hparams.n_ctx_train)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train); |
| 263 | } |
| 264 | |
| 265 | if (cparams.n_ctx_seq > hparams.n_ctx_train) { |
| 266 | LLAMA_LOG_WARN("%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train) |
| 267 | __func__, cparams.n_ctx_seq, hparams.n_ctx_train)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train); |
| 268 | } |
| 269 | |
| 270 | if (!hparams.vocab_only) { |
| 271 | // GPU backends |
| 272 | for (const auto & dev : model.devices) { |
| 273 | ggml_backend_t backend = ggml_backend_dev_init(dev.dev, nullptr); |
| 274 | if (backend == nullptr) { |
| 275 | throwabort_with_suppression(); if (false) std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev.dev))); |
| 276 | } |
| 277 | backends.emplace_back(backend); |
| 278 | } |
| 279 | |
| 280 | // add ACCEL backends (such as BLAS) |
| 281 | for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { |
| 282 | ggml_backend_dev_t dev = ggml_backend_dev_get(i); |
| 283 | if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) { |
| 284 | ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr); |
| 285 | if (backend == nullptr) { |
| 286 | throwabort_with_suppression(); if (false) std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev))); |
| 287 | } |
| 288 | backends.emplace_back(backend); |
| 289 | } |
| 290 | } |
| 291 | |
| 292 | // add CPU backend |
| 293 | backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); |
| 294 | if (backend_cpu == nullptr) { |
| 295 | throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize CPU backend"); |
| 296 | } |
| 297 | backends.emplace_back(backend_cpu); |
| 298 | |
| 299 | // create a list of the set_n_threads functions in the backends |
| 300 | for (auto & backend : backends) { |
| 301 | ggml_backend_dev_t dev = ggml_backend_get_device(backend.get()); |
| 302 | ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; |
| 303 | if (reg) { |
| 304 | auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads"); |
| 305 | if (ggml_backend_set_n_threads_fn) { |
| 306 | set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn); |
| 307 | } |
| 308 | } |
| 309 | } |
| 310 | |
| 311 | llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data); |
| 312 | |
| 313 | // graph outputs buffer |
| 314 | { |
| 315 | if (output_reserve(params.n_seq_max) < params.n_seq_max) { |
| 316 | throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve initial output buffer"); |
| 317 | } |
| 318 | |
| 319 | LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0) |
| 320 | ggml_backend_buffer_name (buf_output.get()),llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0) |
| 321 | ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0); |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | // init the memory module |
| 326 | if (!hparams.vocab_only) { |
| 327 | llama_memory_params params_mem = { |
| 328 | /*.type_k =*/ params.type_k, |
| 329 | /*.type_v =*/ params.type_v, |
| 330 | /*.swa_full =*/ params.swa_full, |
| 331 | /*.ctx_type =*/ cparams.ctx_type, |
| 332 | /*.mem_other =*/ llama_get_memory(cparams.ctx_other), |
| 333 | }; |
| 334 | |
| 335 | memory.reset(model.create_memory(params_mem, cparams)); |
| 336 | } |
| 337 | |
| 338 | // init backends |
| 339 | if (!hparams.vocab_only) { |
| 340 | LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: enumerating backends\n" , __func__); |
| 341 | |
| 342 | backend_buft.clear(); |
| 343 | backend_ptrs.clear(); |
| 344 | backend_buf_exp_size.clear(); |
| 345 | |
| 346 | for (auto & backend : backends) { |
| 347 | auto * buft = ggml_backend_get_default_buffer_type(backend.get()); |
| 348 | auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); |
| 349 | |
| 350 | if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) { |
| 351 | // use the host buffer of the first device CPU for faster transfer of the intermediate state |
| 352 | const auto & dev = model.devices[0]; |
| 353 | auto * host_buft = ggml_backend_dev_host_buffer_type(dev.dev); |
| 354 | if (host_buft) { |
| 355 | buft = host_buft; |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | backend_buft.push_back(buft); |
| 360 | backend_ptrs.push_back(backend.get()); |
| 361 | backend_buf_exp_size.push_back(0); |
| 362 | } |
| 363 | |
| 364 | LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size())llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: backend_ptrs.size() = %zu\n" , __func__, backend_ptrs.size()); |
| 365 | |
| 366 | // TODO: move these checks to ggml_backend_sched |
| 367 | // enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary |
| 368 | bool pipeline_parallel = |
| 369 | model.n_devices() > 1 && |
| 370 | model.n_gpu_layers() > model.hparams.n_layer_all && |
| 371 | model.split_mode() == LLAMA_SPLIT_MODE_LAYER && |
| 372 | cparams.offload_kqv && |
| 373 | !model.has_tensor_overrides(); |
| 374 | |
| 375 | // pipeline parallelism requires support for async compute and events in all devices |
| 376 | if (pipeline_parallel) { |
| 377 | for (auto & backend : backends) { |
| 378 | auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get())); |
| 379 | if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) { |
| 380 | // ignore CPU backend |
| 381 | // TODO: should we ignore ACCEL types too? |
| 382 | continue; |
| 383 | } |
| 384 | auto * dev = ggml_backend_get_device(backend.get()); |
| 385 | ggml_backend_dev_props props; |
| 386 | ggml_backend_dev_get_props(dev, &props); |
| 387 | if (!props.caps.async || !props.caps.events) { |
| 388 | // device does not support async compute or events |
| 389 | pipeline_parallel = false; |
| 390 | break; |
| 391 | } |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | cparams.pipeline_parallel = pipeline_parallel; |
| 396 | |
| 397 | if (cparams.pipeline_parallel) { |
| 398 | LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: pipeline parallelism enabled\n" , __func__); |
| 399 | } |
| 400 | |
| 401 | sched_reserve(); |
| 402 | |
| 403 | if (!cparams.flash_attn) { |
| 404 | if (ggml_is_quantized(params.type_v)) { |
| 405 | throwabort_with_suppression(); if (false) std::runtime_error("quantized V cache was requested, but this requires Flash Attention"); |
| 406 | } |
| 407 | } |
| 408 | } |
| 409 | |
| 410 | // Initialize the full vocabulary token ids for backend samplers. |
| 411 | { |
| 412 | const int n_vocab = model.vocab.n_tokens(); |
| 413 | |
| 414 | sampling.token_ids_full_vocab.resize(n_vocab); |
| 415 | for (int i = 0; i < n_vocab; ++i) { |
| 416 | sampling.token_ids_full_vocab[i] = i; |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | llama_context::~llama_context() { |
| 422 | if (!model.hparams.no_alloc) { |
| 423 | for (size_t i = 0; i < backend_ptrs.size(); ++i) { |
| 424 | ggml_backend_t backend = backend_ptrs[i]; |
| 425 | ggml_backend_buffer_type_t buft = backend_buft[i]; |
| 426 | |
| 427 | const size_t size_exp = backend_buf_exp_size[i]; |
| 428 | const size_t size_act = ggml_backend_sched_get_buffer_size(sched.get(), backend); |
| 429 | if (size_exp == size_act) { |
| 430 | LLAMA_LOG_DEBUG("%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0)) |
| 431 | __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0)); |
| 432 | } else { |
| 433 | LLAMA_LOG_WARN("%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0)) |
| 434 | __func__, ggml_backend_buft_name(buft), size_act / (1024.0*1024.0), size_exp / (1024.0*1024.0))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0)); |
| 435 | } |
| 436 | } |
| 437 | } |
| 438 | ggml_opt_free(opt_ctx); |
| 439 | } |
| 440 | |
| 441 | void llama_context::sched_reserve() { |
| 442 | if (!sched_need_reserve) { |
| 443 | return; |
| 444 | } |
| 445 | |
| 446 | sched_need_reserve = false; |
| 447 | |
| 448 | LLAMA_LOG_INFO("%s: reserving ...\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserving ...\n" , __func__); |
| 449 | |
| 450 | synchronize(); |
| 451 | |
| 452 | const int64_t t_start_us = ggml_time_us(); |
| 453 | |
| 454 | const uint32_t n_seqs = cparams.n_seq_max; |
| 455 | const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); |
| 456 | |
| 457 | const size_t max_nodes = this->graph_max_nodes(n_tokens); |
| 458 | |
| 459 | LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: max_nodes = %zu\n" , __func__, max_nodes); |
| 460 | |
| 461 | gf_res_prev.reset(new llm_graph_result(max_nodes)); |
| 462 | gf_res_reserve.reset(new llm_graph_result(max_nodes)); |
| 463 | |
| 464 | sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload)); |
| 465 | |
| 466 | llama_memory_context_ptr mctx; |
| 467 | if (memory) { |
| 468 | LLAMA_LOG_DEBUG("%s: reserving full memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reserving full memory module\n" , __func__); |
| 469 | mctx = memory->init_full(); |
| 470 | if (!mctx) { |
| 471 | throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize memory module"); |
| 472 | } |
| 473 | } |
| 474 | |
| 475 | // avoid reserving graphs with zero outputs - assume one output per sequence |
| 476 | const int n_outputs = n_seqs; |
| 477 | |
| 478 | LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n" , __func__, n_tokens, n_seqs, n_outputs); |
| 479 | |
| 480 | // resolve automatic Flash Attention use |
| 481 | if (cparams.auto_fa) { |
| 482 | auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true); |
| 483 | if (!gf) { |
| 484 | throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for Flash Attention check"); |
| 485 | } |
| 486 | |
| 487 | const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN"__fattn__") + 1; |
| 488 | bool fa_device_mismatch = false; |
| 489 | for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { |
| 490 | ggml_tensor * n = ggml_graph_node(gf, i); |
| 491 | if (n->op != GGML_OP_FLASH_ATTN_EXT) { |
| 492 | continue; |
| 493 | } |
| 494 | ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); |
| 495 | |
| 496 | // TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer |
| 497 | GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0)if (!(strncmp(n->name, "__fattn__" "-", prefix_len) == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 497, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FATTN \"-\", prefix_len) == 0" ); |
| 498 | const int il = std::stoi(n->name + prefix_len); |
| 499 | ggml_backend_dev_t device_kv = model.dev_layer(il); |
| 500 | if (device_fa != device_kv) { |
| 501 | LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa)) |
| 502 | "is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa)) |
| 503 | __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa)); |
| 504 | // FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways |
| 505 | fa_device_mismatch = true; |
| 506 | break; |
| 507 | } |
| 508 | } |
| 509 | |
| 510 | if (fa_device_mismatch) { |
| 511 | cparams.flash_attn = false; |
| 512 | LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: Flash Attention was auto, set to disabled\n" , __func__); |
| 513 | } else { |
| 514 | cparams.flash_attn = true; |
| 515 | LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: Flash Attention was auto, set to enabled\n" , __func__); |
| 516 | } |
| 517 | |
| 518 | cparams.auto_fa = false; |
| 519 | } |
| 520 | |
| 521 | if (cparams.auto_fgdn) { |
| 522 | LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: resolving fused Gated Delta Net support:\n" , __func__); |
| 523 | |
| 524 | if (cparams.fused_gdn_ar) { |
| 525 | auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true); |
| 526 | if (!gf) { |
| 527 | throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)"); |
| 528 | } |
| 529 | |
| 530 | const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR"__fgdn_ar__") + 1; |
| 531 | bool gdn_device_mismatch = false; |
| 532 | for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { |
| 533 | ggml_tensor * n = ggml_graph_node(gf, i); |
| 534 | if (n->op != GGML_OP_GATED_DELTA_NET) { |
| 535 | continue; |
| 536 | } |
| 537 | ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); |
| 538 | |
| 539 | GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0)if (!(strncmp(n->name, "__fgdn_ar__" "-", prefix_len) == 0 )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 539, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR \"-\", prefix_len) == 0" ); |
| 540 | const int il = std::stoi(n->name + prefix_len); |
| 541 | ggml_backend_dev_t device_kv = model.dev_layer(il); |
| 542 | if (device_gdn != device_kv) { |
| 543 | LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)) |
| 544 | "is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)) |
| 545 | __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)); |
| 546 | gdn_device_mismatch = true; |
| 547 | break; |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | if (gdn_device_mismatch) { |
| 552 | cparams.fused_gdn_ar = false; |
| 553 | LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n" , __func__); |
| 554 | } else { |
| 555 | LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: fused Gated Delta Net (autoregressive) enabled\n" , __func__); |
| 556 | } |
| 557 | } |
| 558 | |
| 559 | if (cparams.fused_gdn_ch) { |
| 560 | // more than one token in the batch per sequence in order to take the chunked path |
| 561 | // note: n_outputs must match n_tokens for embedding models with mean/rank pooling, |
| 562 | // because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies |
| 563 | // it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens, |
| 564 | // the ggml_mul_mat assertion fails. |
| 565 | const uint32_t n_tokens_ch = 16*n_seqs; |
| 566 | auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true); |
| 567 | if (!gf) { |
| 568 | throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)"); |
| 569 | } |
| 570 | |
| 571 | const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH"__fgdn_ch__") + 1; |
| 572 | bool gdn_device_mismatch = false; |
| 573 | for (int i = 0; i < ggml_graph_n_nodes(gf); i++) { |
| 574 | ggml_tensor * n = ggml_graph_node(gf, i); |
| 575 | if (n->op != GGML_OP_GATED_DELTA_NET) { |
| 576 | continue; |
| 577 | } |
| 578 | ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n)); |
| 579 | |
| 580 | GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0)if (!(strncmp(n->name, "__fgdn_ch__" "-", prefix_len) == 0 )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 580, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH \"-\", prefix_len) == 0" ); |
| 581 | const int il = std::stoi(n->name + prefix_len); |
| 582 | ggml_backend_dev_t device_kv = model.dev_layer(il); |
| 583 | if (device_gdn != device_kv) { |
| 584 | LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)) |
| 585 | "is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)) |
| 586 | __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn)); |
| 587 | gdn_device_mismatch = true; |
| 588 | break; |
| 589 | } |
| 590 | } |
| 591 | |
| 592 | if (gdn_device_mismatch) { |
| 593 | cparams.fused_gdn_ch = false; |
| 594 | LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: fused Gated Delta Net (chunked) not supported, set to disabled\n" , __func__); |
| 595 | } else { |
| 596 | LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: fused Gated Delta Net (chunked) enabled\n" , __func__); |
| 597 | } |
| 598 | } |
| 599 | |
| 600 | cparams.auto_fgdn = false; |
| 601 | } |
| 602 | |
| 603 | // reserve worst-case graph |
| 604 | int n_splits_pp = -1; |
| 605 | int n_nodes_pp = -1; |
| 606 | |
| 607 | int n_splits_tg = -1; |
| 608 | int n_nodes_tg = -1; |
| 609 | |
| 610 | const uint32_t n_outputs_pp = std::min(n_tokens, cparams.n_outputs_max); |
| 611 | |
| 612 | // reserve pp (prompt processing) graph first so that buffers are only allocated once |
| 613 | { |
| 614 | auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), |
| 615 | model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr); |
| 616 | if (!gf) { |
| 617 | if (cparams.pipeline_parallel) { |
| 618 | LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: compute buffer allocation failed, retrying without pipeline parallelism\n" , __func__); |
| 619 | cparams.pipeline_parallel = false; |
| 620 | sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload)); |
| 621 | gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get()); |
| 622 | } |
| 623 | if (!gf) { |
| 624 | throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute pp buffers"); |
| 625 | } |
| 626 | } |
| 627 | |
| 628 | n_splits_pp = ggml_backend_sched_get_n_splits(sched.get()); |
| 629 | n_nodes_pp = ggml_graph_n_nodes(gf); |
| 630 | } |
| 631 | |
| 632 | // reserve with tg (token generation) graph to get the number of splits and nodes |
| 633 | { |
| 634 | auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc); |
| 635 | if (!gf) { |
| 636 | throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute tg buffers"); |
| 637 | } |
| 638 | |
| 639 | n_splits_tg = ggml_backend_sched_get_n_splits(sched.get()); |
| 640 | n_nodes_tg = ggml_graph_n_nodes(gf); |
| 641 | } |
| 642 | |
| 643 | // reserve again with pp graph to avoid ggml-alloc reallocations during inference |
| 644 | { |
| 645 | // TODO: not sure if the following graph would be worst case for multi-stream KV caches: |
| 646 | // |
| 647 | // auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get()); |
| 648 | // |
| 649 | auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc); |
| 650 | if (!gf) { |
| 651 | throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute pp buffers"); |
| 652 | } |
| 653 | } |
| 654 | |
| 655 | for (size_t i = 0; i < backend_ptrs.size(); ++i) { |
| 656 | ggml_backend_t backend = backend_ptrs[i]; |
| 657 | ggml_backend_buffer_type_t buft = backend_buft[i]; |
| 658 | if (!model.hparams.no_alloc) { |
| 659 | backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend); |
| 660 | } |
| 661 | if (backend_buf_exp_size[i] > 1) { |
| 662 | LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0) |
| 663 | ggml_backend_buft_name(buft),llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0) |
| 664 | backend_buf_exp_size[i] / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0); |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | if (n_nodes_pp == n_nodes_tg) { |
| 669 | LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph nodes = %d\n" , __func__, n_nodes_pp); |
| 670 | } else { |
| 671 | LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n" , __func__, n_nodes_pp, n_tokens, n_nodes_tg); |
| 672 | } |
| 673 | |
| 674 | if (n_splits_pp == n_splits_tg) { |
| 675 | LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph splits = %d\n" , __func__, n_splits_pp); |
| 676 | } else { |
| 677 | LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph splits = %d (with bs=%d), %d (with bs=1)\n" , __func__, n_splits_pp, n_tokens, n_splits_tg); |
| 678 | } |
| 679 | |
| 680 | const int64_t t_end_us = ggml_time_us(); |
| 681 | |
| 682 | LLAMA_LOG_INFO("%s: reserve took %.2f ms, sched copies = %d\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserve took %.2f ms, sched copies = %d\n" , __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies (sched.get())) |
| 683 | __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserve took %.2f ms, sched copies = %d\n" , __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies (sched.get())); |
| 684 | } |
| 685 | |
| 686 | void llama_context::synchronize() { |
| 687 | if (!sched) { |
| 688 | return; |
| 689 | } |
| 690 | |
| 691 | ggml_backend_sched_synchronize(sched.get()); |
| 692 | |
| 693 | // FIXME: if multiple single tokens are evaluated without a synchronization, |
| 694 | // the stats will be added to the prompt evaluation stats |
| 695 | // this should only happen when using batch size 1 to evaluate a batch |
| 696 | |
| 697 | // add the evaluation to the stats |
| 698 | if (n_queued_tokens == 1) { |
| 699 | if (!cparams.no_perf) { |
| 700 | t_eval_us += ggml_time_us() - t_compute_start_us; |
| 701 | } |
| 702 | n_eval++; |
| 703 | } else if (n_queued_tokens > 1) { |
| 704 | if (!cparams.no_perf) { |
| 705 | t_p_eval_us += ggml_time_us() - t_compute_start_us; |
| 706 | } |
| 707 | n_p_eval += n_queued_tokens; |
| 708 | } |
| 709 | |
| 710 | // get a more accurate load time, upon first eval |
| 711 | if (n_queued_tokens > 0 && !has_evaluated_once) { |
| 712 | t_load_us = ggml_time_us() - t_start_us; |
| 713 | has_evaluated_once = true; |
| 714 | } |
| 715 | |
| 716 | n_queued_tokens = 0; |
| 717 | t_compute_start_us = 0; |
| 718 | } |
| 719 | |
| 720 | const llama_model & llama_context::get_model() const { |
| 721 | return model; |
| 722 | } |
| 723 | |
| 724 | const llama_cparams & llama_context::get_cparams() const { |
| 725 | return cparams; |
| 726 | } |
| 727 | |
| 728 | ggml_backend_sched_t llama_context::get_sched() const { |
| 729 | return sched.get(); |
| 730 | } |
| 731 | |
| 732 | uint32_t llama_context::n_ctx() const { |
| 733 | return cparams.n_ctx; |
| 734 | } |
| 735 | |
| 736 | uint32_t llama_context::n_ctx_seq() const { |
| 737 | return cparams.n_ctx_seq; |
| 738 | } |
| 739 | |
| 740 | uint32_t llama_context::n_batch() const { |
| 741 | return cparams.n_batch; |
| 742 | } |
| 743 | |
| 744 | uint32_t llama_context::n_ubatch() const { |
| 745 | return cparams.n_ubatch; |
| 746 | } |
| 747 | |
| 748 | uint32_t llama_context::n_seq_max() const { |
| 749 | return cparams.n_seq_max; |
| 750 | } |
| 751 | |
| 752 | uint32_t llama_context::n_threads() const { |
| 753 | return cparams.n_threads; |
| 754 | } |
| 755 | |
| 756 | uint32_t llama_context::n_threads_batch() const { |
| 757 | return cparams.n_threads_batch; |
| 758 | } |
| 759 | |
| 760 | llama_memory_t llama_context::get_memory() const { |
| 761 | return memory.get(); |
| 762 | } |
| 763 | |
| 764 | bool llama_context::memory_update(bool optimize) { |
| 765 | if (!memory) { |
| 766 | return false; |
| 767 | } |
| 768 | |
| 769 | { |
| 770 | const auto mctx = memory->init_update(this, optimize); |
| 771 | switch (mctx->get_status()) { |
| 772 | case LLAMA_MEMORY_STATUS_SUCCESS: |
| 773 | { |
| 774 | // noop |
| 775 | } break; |
| 776 | case LLAMA_MEMORY_STATUS_NO_UPDATE: |
| 777 | { |
| 778 | // no updates need to be performed |
| 779 | return false; |
| 780 | } |
| 781 | case LLAMA_MEMORY_STATUS_FAILED_PREPARE: |
| 782 | case LLAMA_MEMORY_STATUS_FAILED_COMPUTE: |
| 783 | { |
| 784 | LLAMA_LOG_ERROR("%s: failed to prepare memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to prepare memory update\n" , __func__); |
| 785 | return false; |
| 786 | } |
| 787 | } |
| 788 | |
| 789 | // reset the previous graph result to make sure that it won't be reused |
| 790 | // TODO: change the mctx->apply() to return information if a graph reserve is needed |
| 791 | // reset the graph result only if the memory module did reset the scheduler |
| 792 | gf_res_prev->reset(); |
| 793 | |
| 794 | if (!mctx->apply()) { |
| 795 | LLAMA_LOG_ERROR("%s: failed to apply memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to apply memory update\n" , __func__); |
| 796 | } |
| 797 | } |
| 798 | |
| 799 | // if the memory module did any computation, we have to reserve a new worst-case graph |
| 800 | { |
| 801 | const auto mctx = memory->init_full(); |
| 802 | if (!mctx) { |
| 803 | throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize memory context"); |
| 804 | } |
| 805 | |
| 806 | const uint32_t n_seqs = cparams.n_seq_max; |
| 807 | const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch); |
| 808 | |
| 809 | const uint32_t n_outputs_max = std::min(n_tokens, cparams.n_outputs_max); |
| 810 | |
| 811 | auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_max, mctx.get()); |
| 812 | if (!gf) { |
| 813 | LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to reserve graph after the memory update\n" , __func__); |
| 814 | } |
| 815 | } |
| 816 | |
| 817 | return true; |
| 818 | } |
| 819 | |
| 820 | enum llama_pooling_type llama_context::pooling_type() const { |
| 821 | return cparams.pooling_type; |
| 822 | } |
| 823 | |
| 824 | float * llama_context::get_logits() { |
| 825 | output_reorder(); |
| 826 | |
| 827 | return logits.data; |
| 828 | } |
| 829 | |
| 830 | int64_t llama_context::output_resolve_row(int32_t i) const { |
| 831 | int64_t j = -1; |
| 832 | |
| 833 | // support negative indices (last output row) |
| 834 | if (i < 0) { |
| 835 | j = n_outputs + i; |
| 836 | if (j < 0) { |
| 837 | throwabort_with_suppression(); if (false) std::runtime_error(format("negative index out of range [0, %d)", n_outputs)); |
| 838 | } |
| 839 | } else if ((size_t) i >= output_ids.size()) { |
| 840 | throwabort_with_suppression(); if (false) std::runtime_error(format("out of range [0, %zu)", output_ids.size())); |
| 841 | } else { |
| 842 | // use output_ids to translate the batch token index into a row number |
| 843 | // that holds this token's data. |
| 844 | j = output_ids[i]; |
| 845 | } |
| 846 | |
| 847 | if (j < 0) { |
| 848 | // the batch token was not configured to output anything |
| 849 | throwabort_with_suppression(); if (false) std::runtime_error(format("batch.logits[%d] != true", i)); |
| 850 | } |
| 851 | |
| 852 | if (j >= n_outputs) { |
| 853 | throwabort_with_suppression(); if (false) std::runtime_error(format("corrupt output buffer (j=%" PRId64"l" "d" ", n_outputs=%d)", j, n_outputs)); |
| 854 | } |
| 855 | |
| 856 | return j; |
| 857 | } |
| 858 | |
| 859 | float * llama_context::get_logits_ith(int32_t i) { |
| 860 | output_reorder(); |
| 861 | |
| 862 | tryif (true) { |
| 863 | if (logits.data == nullptr) { |
| 864 | throwabort_with_suppression(); if (false) std::runtime_error("no logits"); |
| 865 | } |
| 866 | |
| 867 | const int64_t j = output_resolve_row(i); |
| 868 | return logits.data + j*model.vocab.n_tokens(); |
| 869 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 870 | LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid logits id %d, reason: %s\n" , __func__, i, err.what()); |
| 871 | #ifndef NDEBUG |
| 872 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 872, "fatal error"); |
| 873 | #else |
| 874 | return nullptr; |
| 875 | #endif |
| 876 | } |
| 877 | } |
| 878 | |
| 879 | float * llama_context::get_embeddings() { |
| 880 | output_reorder(); |
| 881 | |
| 882 | return embd.data; |
| 883 | } |
| 884 | |
| 885 | llama_token * llama_context::get_sampled_tokens() const{ |
| 886 | return sampling.sampled.data; |
| 887 | } |
| 888 | |
| 889 | float * llama_context::get_embeddings_ith(int32_t i) { |
| 890 | output_reorder(); |
| 891 | |
| 892 | tryif (true) { |
| 893 | if (embd.data == nullptr) { |
| 894 | throwabort_with_suppression(); if (false) std::runtime_error("no embeddings"); |
| 895 | } |
| 896 | |
| 897 | const int64_t j = output_resolve_row(i); |
| 898 | const uint32_t n_embd_out = model.hparams.n_embd_out(); |
| 899 | return embd.data + j*n_embd_out; |
| 900 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 901 | LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid embeddings id %d, reason: %s\n" , __func__, i, err.what()); |
| 902 | #ifndef NDEBUG |
| 903 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 903, "fatal error"); |
| 904 | #else |
| 905 | return nullptr; |
| 906 | #endif |
| 907 | } |
| 908 | } |
| 909 | |
| 910 | float * llama_context::get_embeddings_seq(llama_seq_id seq_id) { |
| 911 | auto it = embd_seq.find(seq_id); |
| 912 | if (it == embd_seq.end()) { |
| 913 | return nullptr; |
| 914 | } |
| 915 | |
| 916 | return it->second.data(); |
| 917 | } |
| 918 | |
| 919 | float * llama_context::get_embeddings_nextn() { |
| 920 | output_reorder(); |
| 921 | |
| 922 | return embd_nextn.data; |
| 923 | } |
| 924 | |
| 925 | float * llama_context::get_embeddings_nextn_ith(int32_t i) { |
| 926 | output_reorder(); |
| 927 | |
| 928 | tryif (true) { |
| 929 | if (embd_nextn.data == nullptr) { |
| 930 | throwabort_with_suppression(); if (false) std::runtime_error("no nextn embeddings"); |
| 931 | } |
| 932 | |
| 933 | const uint32_t n_embd = model.hparams.n_embd_out(); |
| 934 | |
| 935 | if (!cparams.embeddings_nextn_masked) { |
| 936 | // unmasked: nextn rows are stored densely, indexed by raw token position. |
| 937 | if (i < 0 || (size_t)(i + 1) * n_embd > embd_nextn.size) { |
| 938 | throwabort_with_suppression(); if (false) std::runtime_error(format("out of range [0, %zu)", embd_nextn.size / n_embd)); |
| 939 | } |
| 940 | return embd_nextn.data + (size_t) i * n_embd; |
| 941 | } |
| 942 | |
| 943 | const int64_t j = output_resolve_row(i); |
| 944 | return embd_nextn.data + j*n_embd; |
| 945 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 946 | LLAMA_LOG_ERROR("%s: invalid nextn embeddings id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid nextn embeddings id %d, reason: %s\n" , __func__, i, err.what()); |
| 947 | #ifndef NDEBUG |
| 948 | GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 948, "fatal error"); |
| 949 | #else |
| 950 | return nullptr; |
| 951 | #endif |
| 952 | } |
| 953 | } |
| 954 | |
| 955 | float * llama_context::get_embeddings_layer_inp(uint32_t lid) { |
| 956 | output_reorder(); |
| 957 | |
| 958 | GGML_ASSERT(lid < embd_layer_inp.size() && embd_layer_inp[lid].has_data())if (!(lid < embd_layer_inp.size() && embd_layer_inp [lid].has_data())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 958, "GGML_ASSERT(%s) failed", "lid < embd_layer_inp.size() && embd_layer_inp[lid].has_data()" ); |
| 959 | |
| 960 | return embd_layer_inp[lid].data; |
| 961 | } |
| 962 | |
| 963 | llama_token llama_context::get_sampled_token_ith(int32_t idx) { |
| 964 | output_reorder(); |
| 965 | |
| 966 | if (!sampling.sampled.has_data()) { |
| 967 | return LLAMA_TOKEN_NULL-1; |
| 968 | } |
| 969 | |
| 970 | tryif (true) { |
| 971 | const int64_t row = output_resolve_row(idx); |
| 972 | GGML_ASSERT(row < (int64_t) sampling.sampled.size)if (!(row < (int64_t) sampling.sampled.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 972, "GGML_ASSERT(%s) failed", "row < (int64_t) sampling.sampled.size" ); |
| 973 | return sampling.sampled.data[row]; |
| 974 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 975 | LLAMA_LOG_ERROR("%s: invalid backend sampled token id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled token id %d, reason: %s\n" , __func__, idx, err.what()); |
| 976 | return LLAMA_TOKEN_NULL-1; |
| 977 | } |
| 978 | } |
| 979 | |
| 980 | float * llama_context::get_sampled_probs_ith(int32_t idx) { |
| 981 | output_reorder(); |
| 982 | |
| 983 | if (!sampling.probs.has_data()) { |
| 984 | return nullptr; |
| 985 | } |
| 986 | |
| 987 | tryif (true) { |
| 988 | const int64_t row = output_resolve_row(idx); |
| 989 | if ((size_t) row >= sampling.probs_count.size() || sampling.probs_count[row] == 0) { |
| 990 | return nullptr; |
| 991 | } |
| 992 | return sampling.probs.data + row*model.vocab.n_tokens(); |
| 993 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 994 | LLAMA_LOG_ERROR("%s: invalid backend sampled probs id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled probs id %d, reason: %s\n" , __func__, idx, err.what()); |
| 995 | return nullptr; |
| 996 | } |
| 997 | } |
| 998 | |
| 999 | float * llama_context::get_sampled_logits_ith(int32_t idx) { |
| 1000 | output_reorder(); |
| 1001 | |
| 1002 | if (!sampling.logits.has_data()) { |
| 1003 | return nullptr; |
| 1004 | } |
| 1005 | |
| 1006 | tryif (true) { |
| 1007 | const int64_t row = output_resolve_row(idx); |
| 1008 | if ((size_t) row >= sampling.logits_count.size() || sampling.logits_count[row] == 0) { |
| 1009 | return nullptr; |
| 1010 | } |
| 1011 | return sampling.logits.data + row*model.vocab.n_tokens(); |
| 1012 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 1013 | LLAMA_LOG_ERROR("%s: invalid backend sampled logits id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled logits id %d, reason: %s\n" , __func__, idx, err.what()); |
| 1014 | return nullptr; |
| 1015 | } |
| 1016 | } |
| 1017 | |
| 1018 | const llama_token * llama_context::get_sampled_candidates_ith(int32_t idx) { |
| 1019 | output_reorder(); |
| 1020 | |
| 1021 | tryif (true) { |
| 1022 | const int64_t row = output_resolve_row(idx); |
| 1023 | if (sampling.candidates.has_data() && |
| 1024 | (size_t) row < sampling.candidates_count.size() && |
| 1025 | sampling.candidates_count[row] > 0) { |
| 1026 | return sampling.candidates.data + row*model.vocab.n_tokens(); |
| 1027 | } |
| 1028 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 1029 | // fallback to full vocab list |
| 1030 | GGML_UNUSED(err)(void)(err); |
| 1031 | } |
| 1032 | |
| 1033 | return sampling.token_ids_full_vocab.data(); |
| 1034 | } |
| 1035 | |
| 1036 | size_t llama_context::get_sampled_candidates_count(int32_t idx) { |
| 1037 | output_reorder(); |
| 1038 | |
| 1039 | if (!sampling.candidates.has_data()) { |
| 1040 | return 0; |
| 1041 | } |
| 1042 | |
| 1043 | tryif (true) { |
| 1044 | const int64_t row = output_resolve_row(idx); |
| 1045 | if ((size_t) row >= sampling.candidates_count.size()) { |
| 1046 | return 0; |
| 1047 | } |
| 1048 | return sampling.candidates_count[row]; |
| 1049 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 1050 | LLAMA_LOG_ERROR("%s: invalid backend sampled candidates count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled candidates count id %d, reason: %s\n" , __func__, idx, err.what()); |
| 1051 | return 0; |
| 1052 | } |
| 1053 | } |
| 1054 | |
| 1055 | size_t llama_context::get_sampled_logits_count(int32_t idx) { |
| 1056 | output_reorder(); |
| 1057 | |
| 1058 | if (!sampling.logits.has_data()) { |
| 1059 | return model.vocab.n_tokens(); |
| 1060 | } |
| 1061 | |
| 1062 | tryif (true) { |
| 1063 | const int64_t row = output_resolve_row(idx); |
| 1064 | if ((size_t) row >= sampling.logits_count.size()) { |
| 1065 | return 0; |
| 1066 | } |
| 1067 | return sampling.logits_count[row]; |
| 1068 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 1069 | LLAMA_LOG_ERROR("%s: invalid backend sampled logits count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled logits count id %d, reason: %s\n" , __func__, idx, err.what()); |
| 1070 | return 0; |
| 1071 | } |
| 1072 | } |
| 1073 | |
| 1074 | size_t llama_context::get_sampled_probs_count(int32_t idx) { |
| 1075 | output_reorder(); |
| 1076 | |
| 1077 | if (!sampling.probs.has_data()) { |
| 1078 | return 0; |
| 1079 | } |
| 1080 | |
| 1081 | tryif (true) { |
| 1082 | const int64_t row = output_resolve_row(idx); |
| 1083 | if ((size_t) row >= sampling.probs_count.size()) { |
| 1084 | return 0; |
| 1085 | } |
| 1086 | return sampling.probs_count[row]; |
| 1087 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 1088 | LLAMA_LOG_ERROR("%s: invalid backend sampled probs count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled probs count id %d, reason: %s\n" , __func__, idx, err.what()); |
| 1089 | return 0; |
| 1090 | } |
| 1091 | } |
| 1092 | |
| 1093 | |
| 1094 | void llama_context::attach_threadpool( |
| 1095 | ggml_threadpool_t threadpool, |
| 1096 | ggml_threadpool_t threadpool_batch) { |
| 1097 | LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ ); |
| 1098 | |
| 1099 | this->threadpool = threadpool; |
| 1100 | this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool; |
| 1101 | } |
| 1102 | |
| 1103 | void llama_context::detach_threadpool() { |
| 1104 | LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ ); |
| 1105 | |
| 1106 | this->threadpool = nullptr; |
| 1107 | this->threadpool_batch = nullptr; |
| 1108 | } |
| 1109 | |
| 1110 | void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) { |
| 1111 | LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_threads = %d, n_threads_batch = %d\n" , __func__, n_threads, n_threads_batch); |
| 1112 | |
| 1113 | cparams.n_threads = n_threads; |
| 1114 | cparams.n_threads_batch = n_threads_batch; |
| 1115 | } |
| 1116 | |
| 1117 | void llama_context::set_abort_callback(bool (*abort_callback)(void * data), void * abort_callback_data) { |
| 1118 | LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ ); |
| 1119 | |
| 1120 | this->abort_callback = abort_callback; |
| 1121 | this->abort_callback_data = abort_callback_data; |
| 1122 | |
| 1123 | for (auto & backend : backends) { |
| 1124 | auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get())); |
| 1125 | if (reg) { |
| 1126 | auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback"); |
| 1127 | if (set_abort_callback_fn) { |
| 1128 | set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data); |
| 1129 | } |
| 1130 | } |
| 1131 | } |
| 1132 | } |
| 1133 | |
| 1134 | void llama_context::set_embeddings(bool value) { |
| 1135 | LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value); |
| 1136 | |
| 1137 | cparams.embeddings = value; |
| 1138 | |
| 1139 | // TODO: not sure yet if we want to reserve here |
| 1140 | //sched_need_reserve = true; |
| 1141 | } |
| 1142 | |
| 1143 | void llama_context::set_embeddings_nextn(bool value, bool masked) { |
| 1144 | LLAMA_LOG_DEBUG("%s: value = %d, masked = %d\n", __func__, value, masked)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d, masked = %d\n" , __func__, value, masked); |
| 1145 | |
| 1146 | cparams.embeddings_nextn = value; |
| 1147 | cparams.embeddings_nextn_masked = masked; |
| 1148 | } |
| 1149 | |
| 1150 | void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) { |
| 1151 | LLAMA_LOG_DEBUG("%s: lid = %d, enable = %d\n", __func__, lid, enable)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: lid = %d, enable = %d\n" , __func__, lid, enable); |
| 1152 | |
| 1153 | GGML_ASSERT(lid < model.hparams.n_layer())if (!(lid < model.hparams.n_layer())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1153, "GGML_ASSERT(%s) failed", "lid < model.hparams.n_layer()" ); |
| 1154 | |
| 1155 | cparams.embeddings_layer_inp[lid] = enable; |
| 1156 | |
| 1157 | // note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected |
| 1158 | sched_need_reserve = true; |
| 1159 | } |
| 1160 | |
| 1161 | void llama_context::set_causal_attn(bool value) { |
| 1162 | LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value); |
| 1163 | |
| 1164 | if (cparams.causal_attn == value) { |
| 1165 | return; |
| 1166 | } |
| 1167 | |
| 1168 | cparams.causal_attn = value; |
| 1169 | |
| 1170 | sched_need_reserve = true; |
| 1171 | } |
| 1172 | |
| 1173 | void llama_context::set_warmup(bool value) { |
| 1174 | LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value); |
| 1175 | |
| 1176 | if (cparams.warmup == value) { |
| 1177 | return; |
| 1178 | } |
| 1179 | |
| 1180 | cparams.warmup = value; |
| 1181 | |
| 1182 | // warmups are usually with small batches, so no need to reserve |
| 1183 | //sched_need_reserve = true; |
| 1184 | } |
| 1185 | |
| 1186 | bool llama_context::set_sampler(llama_seq_id seq_id, llama_sampler * sampler) { |
| 1187 | if (!sampler && sampling.samplers.count(seq_id) == 0) { |
| 1188 | return true; |
| 1189 | } |
| 1190 | |
| 1191 | LLAMA_LOG_DEBUG("%s: seq_id = %d, sampler = %p\n", __func__, (int) seq_id, (void *) sampler)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: seq_id = %d, sampler = %p\n" , __func__, (int) seq_id, (void *) sampler); |
| 1192 | |
| 1193 | if (sampler && model.split_mode() == LLAMA_SPLIT_MODE_TENSOR) { |
| 1194 | static bool warned = false; |
| 1195 | if (!warned) { |
| 1196 | LLAMA_LOG_WARN("%s: backend sampling not supported with SPLIT_MODE_TENSOR; using CPU\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: backend sampling not supported with SPLIT_MODE_TENSOR; using CPU\n" , __func__); |
| 1197 | warned = true; |
| 1198 | } |
| 1199 | if (sampling.samplers.count(seq_id) > 0) { |
| 1200 | sched_need_reserve = true; |
| 1201 | } |
| 1202 | sampling.samplers.erase(seq_id); |
| 1203 | return false; |
| 1204 | } |
| 1205 | |
| 1206 | const bool can_offload = |
| 1207 | sampler && |
| 1208 | sampler->iface->backend_init && |
| 1209 | sampler->iface->backend_apply && |
| 1210 | llama_sampler_chain_n(sampler) > 0; |
| 1211 | |
| 1212 | if (sampler && can_offload) { |
| 1213 | auto * buft = ggml_backend_dev_buffer_type(model.dev_output()); |
| 1214 | |
| 1215 | sampler->iface->backend_init(sampler, buft); |
| 1216 | |
| 1217 | sampling.samplers[seq_id] = sampler; |
| 1218 | |
| 1219 | sched_need_reserve = true; |
| 1220 | |
| 1221 | return true; |
| 1222 | } |
| 1223 | |
| 1224 | if (sampler && !can_offload) { |
| 1225 | LLAMA_LOG_WARN("%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\n", __func__, llama_sampler_name(sampler), seq_id)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\n" , __func__, llama_sampler_name(sampler), seq_id); |
| 1226 | |
| 1227 | if (sampling.samplers.count(seq_id) > 0) { |
| 1228 | sched_need_reserve = true; |
| 1229 | } |
| 1230 | |
| 1231 | sampling.samplers.erase(seq_id); |
| 1232 | |
| 1233 | return false; |
| 1234 | } |
| 1235 | |
| 1236 | sampling.samplers.erase(seq_id); |
| 1237 | |
| 1238 | sched_need_reserve = true; |
| 1239 | |
| 1240 | return true; |
| 1241 | } |
| 1242 | |
| 1243 | void llama_context::set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) { |
| 1244 | LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void *) adapters)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: adapters = %p\n" , __func__, (void *) adapters); |
| 1245 | |
| 1246 | if (adapters_lora_are_same(adapters, n_adapters, scales)) { |
| 1247 | return; |
| 1248 | } |
| 1249 | |
| 1250 | loras.reset(new llama_adapter_loras()); |
| 1251 | |
| 1252 | for (size_t i = 0; i < n_adapters; i ++) { |
| 1253 | if (scales[i] != 0.0f) { |
| 1254 | loras->insert({adapters[i], scales[i]}); |
| 1255 | } |
| 1256 | } |
| 1257 | |
| 1258 | sched_need_reserve = true; |
| 1259 | } |
| 1260 | |
| 1261 | bool llama_context::adapters_lora_are_same(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) { |
| 1262 | LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void *) adapters)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: adapters = %p\n" , __func__, (void *) adapters); |
| 1263 | |
| 1264 | // Adapters with a zero scale are never added to `loras`, so also ignore them for the comparison. |
| 1265 | size_t n_non_zero = 0; |
| 1266 | |
| 1267 | for (size_t i = 0; i < n_adapters; i ++) { |
| 1268 | if (scales[i] == 0.0f) { |
| 1269 | continue; |
| 1270 | } |
| 1271 | n_non_zero++; |
| 1272 | |
| 1273 | auto it = loras->find(adapters[i]); |
| 1274 | |
| 1275 | if (it == loras->end() || it->second != scales[i]) { |
| 1276 | return false; |
| 1277 | } |
| 1278 | } |
| 1279 | |
| 1280 | if (n_non_zero != loras->size()) { |
| 1281 | return false; |
| 1282 | } |
| 1283 | |
| 1284 | return true; |
| 1285 | } |
| 1286 | |
| 1287 | bool llama_context::set_adapter_cvec( |
| 1288 | const float * data, |
| 1289 | size_t len, |
| 1290 | int32_t n_embd, |
| 1291 | int32_t il_start, |
| 1292 | int32_t il_end) { |
| 1293 | LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: il_start = %d, il_end = %d\n" , __func__, il_start, il_end); |
| 1294 | |
| 1295 | bool res = cvec->apply(model, data, len, n_embd, il_start, il_end); |
| 1296 | |
| 1297 | sched_need_reserve = true; |
| 1298 | |
| 1299 | return res; |
| 1300 | } |
| 1301 | |
| 1302 | llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) { |
| 1303 | if (mctx && !mctx->apply()) { |
| 1304 | LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to apply memory context\n" , __func__); |
| 1305 | ret = GGML_STATUS_FAILED; |
| 1306 | return nullptr; |
| 1307 | } |
| 1308 | |
| 1309 | auto * res = gf_res_prev.get(); |
| 1310 | auto * gf = res->get_gf(); |
Value stored to 'gf' during its initialization is never read | |
| 1311 | |
| 1312 | // the new graph parameters |
| 1313 | // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters |
| 1314 | const auto gparams = graph_params(res, ubatch, mctx, gtype); |
| 1315 | |
| 1316 | if (!graph_reuse_disable && res->can_reuse(gparams)) { |
| 1317 | //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__); |
| 1318 | |
| 1319 | // with pipeline parallelism, the previous graph_compute_async may still be running |
| 1320 | // on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors |
| 1321 | // that the previous compute is still reading. |
| 1322 | if (cparams.pipeline_parallel) { |
| 1323 | ggml_backend_sched_synchronize(sched.get()); |
| 1324 | } |
| 1325 | |
| 1326 | n_reused++; |
| 1327 | } else { |
| 1328 | res->reset(); |
| 1329 | |
| 1330 | ggml_backend_sched_reset(sched.get()); |
| 1331 | ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data); |
| 1332 | |
| 1333 | //const auto t_start_us = ggml_time_us(); |
| 1334 | |
| 1335 | gf = model.build_graph(gparams); |
| 1336 | |
| 1337 | //LLAMA_LOG_INFO("graph build time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0); |
| 1338 | |
| 1339 | if (!gf) { |
| 1340 | LLAMA_LOG_ERROR("%s: failed to initialize graph\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize graph\n" , __func__); |
| 1341 | ret = GGML_STATUS_FAILED; |
| 1342 | return nullptr; |
| 1343 | } |
| 1344 | |
| 1345 | if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) { |
| 1346 | LLAMA_LOG_ERROR("%s: failed to allocate graph\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate graph\n" , __func__); |
| 1347 | ret = GGML_STATUS_ALLOC_FAILED; |
| 1348 | return nullptr; |
| 1349 | } |
| 1350 | } |
| 1351 | |
| 1352 | // set the input data for the input tensors |
| 1353 | { |
| 1354 | //const auto t_start_us = ggml_time_us(); |
| 1355 | |
| 1356 | // FIXME this call causes a crash if any model inputs were not used in the graph and were therefore not allocated |
| 1357 | res->set_inputs(&ubatch); |
| 1358 | |
| 1359 | //LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0); |
| 1360 | } |
| 1361 | |
| 1362 | const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1); |
| 1363 | if (status != GGML_STATUS_SUCCESS) { |
| 1364 | LLAMA_LOG_ERROR("%s: failed to compute graph, compute status: %d\n", __func__, status)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to compute graph, compute status: %d\n" , __func__, status); |
| 1365 | ret = status; |
| 1366 | return nullptr; |
| 1367 | } |
| 1368 | |
| 1369 | ret = GGML_STATUS_SUCCESS; |
| 1370 | |
| 1371 | return res; |
| 1372 | } |
| 1373 | |
| 1374 | int llama_context::encode(const llama_batch & batch_inp) { |
| 1375 | // MTP hook batches carry both token (next-token id) and embd (h_nextn row), |
| 1376 | // so accept either present rather than requiring exactly one. |
| 1377 | GGML_ASSERT(batch_inp.token || batch_inp.embd)if (!(batch_inp.token || batch_inp.embd)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1377, "GGML_ASSERT(%s) failed", "batch_inp.token || batch_inp.embd" ); |
| 1378 | |
| 1379 | if (batch_inp.n_tokens == 0) { |
| 1380 | LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_tokens == 0\n" , __func__); |
| 1381 | return -1; |
| 1382 | } |
| 1383 | |
| 1384 | const auto & hparams = model.hparams; |
| 1385 | |
| 1386 | // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim |
| 1387 | const int64_t n_embd = hparams.n_embd_inp(); |
| 1388 | const int64_t n_vocab = model.vocab.n_tokens(); |
| 1389 | |
| 1390 | // note: during encode, we always pass the full sequence starting from pos = 0 |
| 1391 | if (!balloc->init(batch_inp, model.vocab, nullptr, n_embd, cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max, true)) { |
| 1392 | LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__); |
| 1393 | return -1; |
| 1394 | } |
| 1395 | |
| 1396 | const uint32_t n_tokens = balloc->get_n_tokens(); |
| 1397 | |
| 1398 | // [TAG_NO_CACHE_PAD] |
| 1399 | // TODO: add new split mode where we pad the input sequences so that ubatch.equal_seqs == true |
| 1400 | const llama_ubatch ubatch = balloc->split_simple(n_tokens); |
| 1401 | |
| 1402 | // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot |
| 1403 | GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens")if (!(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1403, "GGML_ASSERT(%s) failed", "cparams.n_ubatch >= n_tokens && \"encoder requires n_ubatch >= n_tokens\"" ); |
| 1404 | |
| 1405 | if (t_compute_start_us == 0) { |
| 1406 | t_compute_start_us = ggml_time_us(); |
| 1407 | } |
| 1408 | |
| 1409 | // TODO: this clear of the buffer can easily be forgotten - need something better |
| 1410 | embd_seq.clear(); |
| 1411 | |
| 1412 | sched_reserve(); |
| 1413 | |
| 1414 | n_queued_tokens += n_tokens; |
| 1415 | |
| 1416 | // reserve output buffer |
| 1417 | if (output_reserve(n_tokens) < n_tokens) { |
| 1418 | LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %u outputs\n" , __func__, n_tokens); |
| 1419 | return -2; |
| 1420 | }; |
| 1421 | |
| 1422 | for (uint32_t i = 0; i < n_tokens; ++i) { |
| 1423 | output_ids[i] = i; |
| 1424 | } |
| 1425 | |
| 1426 | n_outputs = n_tokens; |
| 1427 | |
| 1428 | const auto causal_attn_org = cparams.causal_attn; |
| 1429 | |
| 1430 | // always use non-causal attention for encoder graphs |
| 1431 | // TODO: this is a tmp solution until we have a proper way to support enc-dec models |
| 1432 | // ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223 |
| 1433 | cparams.causal_attn = false; |
| 1434 | |
| 1435 | ggml_status status; |
| 1436 | const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_ENCODER, nullptr, status); |
| 1437 | |
| 1438 | cparams.causal_attn = causal_attn_org; |
| 1439 | |
| 1440 | if (!res) { |
| 1441 | switch (status) { |
| 1442 | case GGML_STATUS_ABORTED: return 2; |
| 1443 | case GGML_STATUS_ALLOC_FAILED: return -2; |
| 1444 | case GGML_STATUS_FAILED: return -3; |
| 1445 | case GGML_STATUS_SUCCESS: GGML_ABORT("should not happen")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1445, "should not happen"); |
| 1446 | } |
| 1447 | } |
| 1448 | |
| 1449 | auto * t_logits = res->get_logits(); |
| 1450 | auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd(); |
| 1451 | auto * t_h_nextn = cparams.embeddings_nextn ? res->get_h_nextn() : nullptr; |
| 1452 | |
| 1453 | // extract logits |
| 1454 | if (logits.data && t_logits) { |
| 1455 | ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits); |
| 1456 | GGML_ASSERT(backend_res != nullptr)if (!(backend_res != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1456, "GGML_ASSERT(%s) failed", "backend_res != nullptr"); |
| 1457 | GGML_ASSERT(logits.data != nullptr)if (!(logits.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1457, "GGML_ASSERT(%s) failed", "logits.data != nullptr"); |
| 1458 | |
| 1459 | ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float)); |
| 1460 | } |
| 1461 | |
| 1462 | // extract embeddings |
| 1463 | if (embd.data && t_embd) { |
| 1464 | ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd); |
| 1465 | GGML_ASSERT(backend_embd != nullptr)if (!(backend_embd != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1465, "GGML_ASSERT(%s) failed", "backend_embd != nullptr"); |
| 1466 | |
| 1467 | switch (cparams.pooling_type) { |
| 1468 | case LLAMA_POOLING_TYPE_NONE: |
| 1469 | { |
| 1470 | // extract token embeddings |
| 1471 | GGML_ASSERT(embd.data != nullptr)if (!(embd.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1471, "GGML_ASSERT(%s) failed", "embd.data != nullptr"); |
| 1472 | const uint32_t n_embd_out = hparams.n_embd_out(); |
| 1473 | |
| 1474 | GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size)if (!(n_tokens*n_embd_out <= (int64_t) embd.size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1474, "GGML_ASSERT(%s) failed", "n_tokens*n_embd_out <= (int64_t) embd.size" ); |
| 1475 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float)); |
| 1476 | } break; |
| 1477 | case LLAMA_POOLING_TYPE_MEAN: |
| 1478 | case LLAMA_POOLING_TYPE_CLS: |
| 1479 | case LLAMA_POOLING_TYPE_LAST: |
| 1480 | { |
| 1481 | // extract sequence embeddings |
| 1482 | auto & embd_seq_out = embd_seq; |
| 1483 | |
| 1484 | for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { |
| 1485 | const llama_seq_id seq_id = ubatch.seq_id_unq[s]; |
| 1486 | const int32_t seq_idx = ubatch.seq_idx[seq_id]; |
| 1487 | |
| 1488 | // use n_embd_out (not n_embd_inp) - the pooled embedding has the model's |
| 1489 | // output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl) |
| 1490 | const uint32_t n_embd_out = hparams.n_embd_out(); |
| 1491 | embd_seq_out[seq_id].resize(n_embd_out); |
| 1492 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_out*seq_idx)*sizeof(float), n_embd_out*sizeof(float)); |
| 1493 | } |
| 1494 | } break; |
| 1495 | case LLAMA_POOLING_TYPE_RANK: |
| 1496 | { |
| 1497 | // extract the rerank score - n_cls_out floats per sequence |
| 1498 | auto & embd_seq_out = embd_seq; |
| 1499 | |
| 1500 | const uint32_t n_cls_out = hparams.n_cls_out; |
| 1501 | |
| 1502 | for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { |
| 1503 | const llama_seq_id seq_id = ubatch.seq_id_unq[s]; |
| 1504 | const int32_t seq_idx = ubatch.seq_idx[seq_id]; |
| 1505 | |
| 1506 | embd_seq_out[seq_id].resize(n_cls_out); |
| 1507 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_out*seq_idx)*sizeof(float), n_cls_out*sizeof(float)); |
| 1508 | } |
| 1509 | } break; |
| 1510 | case LLAMA_POOLING_TYPE_UNSPECIFIED: |
| 1511 | { |
| 1512 | GGML_ABORT("unknown pooling type")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1512, "unknown pooling type"); |
| 1513 | } |
| 1514 | } |
| 1515 | } |
| 1516 | |
| 1517 | // extract nextn embeddings (hidden state before the final output norm) |
| 1518 | if (embd_nextn.data && t_h_nextn && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { |
| 1519 | ggml_backend_t backend_h = ggml_backend_sched_get_tensor_backend(sched.get(), t_h_nextn); |
| 1520 | GGML_ASSERT(backend_h != nullptr)if (!(backend_h != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1520, "GGML_ASSERT(%s) failed", "backend_h != nullptr"); |
| 1521 | |
| 1522 | const uint32_t n_embd = hparams.n_embd_out(); |
| 1523 | GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_nextn.size)if (!(n_tokens*n_embd <= (int64_t) embd_nextn.size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1523, "GGML_ASSERT(%s) failed", "n_tokens*n_embd <= (int64_t) embd_nextn.size" ); |
| 1524 | ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokens*n_embd*sizeof(float)); |
| 1525 | } |
| 1526 | |
| 1527 | // TODO: hacky solution |
| 1528 | if (model.arch == LLM_ARCH_T5 && t_embd) { |
| 1529 | //cross.t_embd = t_embd; |
| 1530 | |
| 1531 | synchronize(); |
| 1532 | |
| 1533 | cross.n_embd = t_embd->ne[0]; |
| 1534 | cross.n_enc = t_embd->ne[1]; |
| 1535 | cross.v_embd.resize(cross.n_embd*cross.n_enc); |
| 1536 | memcpy(cross.v_embd.data(), embd.data, ggml_nbytes(t_embd)); |
| 1537 | |
| 1538 | const auto & batch = balloc->get_batch(); |
| 1539 | |
| 1540 | // remember the sequence ids used during the encoding - needed for cross attention later |
| 1541 | cross.seq_ids_enc.resize(n_tokens); |
| 1542 | for (uint32_t i = 0; i < n_tokens; i++) { |
| 1543 | cross.seq_ids_enc[i].clear(); |
| 1544 | |
| 1545 | for (int s = 0; s < batch.n_seq_id[i]; s++) { |
| 1546 | const llama_seq_id seq_id = batch.seq_id[i][s]; |
| 1547 | |
| 1548 | cross.seq_ids_enc[i].insert(seq_id); |
| 1549 | } |
| 1550 | } |
| 1551 | } |
| 1552 | |
| 1553 | return 0; |
| 1554 | } |
| 1555 | |
| 1556 | static std::map<llama_seq_id, uint32_t> build_seq_to_output_row(const llama_ubatch & ubatch, uint32_t row_offset) { |
| 1557 | std::map<llama_seq_id, uint32_t> seq_to_row; |
| 1558 | // how many output tokens we have seen so far for this ubatch. |
| 1559 | uint32_t local = 0; |
| 1560 | for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { |
| 1561 | // skip tokens that are not output. |
| 1562 | if (!ubatch.output[i]) { |
| 1563 | continue; |
| 1564 | } |
| 1565 | |
| 1566 | const llama_seq_id seq_id = ubatch.seq_id[i][0]; |
| 1567 | // row_offset is the number of output tokens before this ubatch. |
| 1568 | seq_to_row[seq_id] = row_offset + local; |
| 1569 | ++local; |
| 1570 | } |
| 1571 | return seq_to_row; |
| 1572 | } |
| 1573 | |
| 1574 | static void copy_tensor_async_ints( |
| 1575 | const std::map<llama_seq_id, ggml_tensor*> & tensor_map, |
| 1576 | const buffer_view<llama_token> & sampled, |
| 1577 | const std::map<llama_seq_id, uint32_t> & seq_to_row, |
| 1578 | ggml_backend_sched_t sched) { |
| 1579 | if (!sampled.has_data()) { |
| 1580 | return; |
| 1581 | } |
| 1582 | |
| 1583 | for (const auto & [seq_id, tensor] : tensor_map) { |
| 1584 | auto it = seq_to_row.find(seq_id); |
| 1585 | if (it == seq_to_row.end()) { |
| 1586 | continue; |
| 1587 | } |
| 1588 | |
| 1589 | const uint32_t row = it->second; |
| 1590 | GGML_ASSERT(row < sampled.size)if (!(row < sampled.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1590, "GGML_ASSERT(%s) failed", "row < sampled.size"); |
| 1591 | |
| 1592 | GGML_ASSERT(ggml_is_contiguous(tensor) && "sampled tokens tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "sampled tokens tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1592, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"sampled tokens tensor must be contiguous for async copy\"" ); |
| 1593 | |
| 1594 | ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor); |
| 1595 | ggml_backend_tensor_get_async(backend, tensor, sampled.data + row, 0, sizeof(sampled.data[row])); |
| 1596 | } |
| 1597 | } |
| 1598 | |
| 1599 | static void copy_tensor_async_floats( |
| 1600 | const std::map<llama_seq_id, ggml_tensor*> & tensor_map, |
| 1601 | const buffer_view<float> & dst, |
| 1602 | size_t stride, |
| 1603 | std::vector<uint32_t> & counts, |
| 1604 | const std::map<llama_seq_id, uint32_t> & seq_to_row, |
| 1605 | ggml_backend_sched_t sched) { |
| 1606 | if (!dst.has_data()) { |
| 1607 | return; |
| 1608 | } |
| 1609 | |
| 1610 | for (const auto & [seq_id, tensor] : tensor_map) { |
| 1611 | auto it = seq_to_row.find(seq_id); |
| 1612 | if (it == seq_to_row.end()) { |
| 1613 | continue; |
| 1614 | } |
| 1615 | |
| 1616 | const uint32_t row = it->second; |
| 1617 | GGML_ASSERT(row < counts.size())if (!(row < counts.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1617, "GGML_ASSERT(%s) failed", "row < counts.size()"); |
| 1618 | |
| 1619 | GGML_ASSERT(ggml_is_contiguous(tensor) && "logits/probs tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "logits/probs tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1619, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"logits/probs tensor must be contiguous for async copy\"" ); |
| 1620 | |
| 1621 | ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor); |
| 1622 | float * row_ptr = dst.data + (size_t) row * stride; |
| 1623 | ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor)); |
| 1624 | |
| 1625 | // Update the actual number of logits/probabilities that were written for this row. |
| 1626 | counts[row] = ggml_nelements(tensor); |
| 1627 | } |
| 1628 | } |
| 1629 | |
| 1630 | static void copy_tensor_async_candidates( |
| 1631 | const std::map<llama_seq_id, ggml_tensor*> & tensor_map, |
| 1632 | const buffer_view<llama_token> & dst, |
| 1633 | size_t stride, |
| 1634 | std::vector<uint32_t> & counts, |
| 1635 | const std::map<llama_seq_id, uint32_t> & seq_to_row, |
| 1636 | ggml_backend_sched_t sched) { |
| 1637 | if (!dst.has_data()) { |
| 1638 | return; |
| 1639 | } |
| 1640 | |
| 1641 | for (const auto & [seq_id, tensor] : tensor_map) { |
| 1642 | auto it = seq_to_row.find(seq_id); |
| 1643 | if (it == seq_to_row.end()) { |
| 1644 | continue; |
| 1645 | } |
| 1646 | |
| 1647 | const uint32_t row = it->second; |
| 1648 | GGML_ASSERT(row < counts.size())if (!(row < counts.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1648, "GGML_ASSERT(%s) failed", "row < counts.size()"); |
| 1649 | |
| 1650 | GGML_ASSERT(ggml_is_contiguous(tensor) && "candidates tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "candidates tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1650, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"candidates tensor must be contiguous for async copy\"" ); |
| 1651 | |
| 1652 | ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor); |
| 1653 | llama_token * row_ptr = dst.data + (size_t) row * stride; |
| 1654 | ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor)); |
| 1655 | |
| 1656 | // Update the actual number of candidates that were written. |
| 1657 | counts[row] = ggml_nelements(tensor); |
| 1658 | } |
| 1659 | } |
| 1660 | |
| 1661 | static bool needs_raw_logits(const llama_ubatch & ubatch, const std::map<llama_seq_id, llama_sampler *> & samplers) { |
| 1662 | for (uint32_t i = 0; i < ubatch.n_tokens; i++) { |
| 1663 | if (!ubatch.output[i]) { |
| 1664 | continue; |
| 1665 | } |
| 1666 | |
| 1667 | // Check if the output token has at least one sequence without a backend sampler. |
| 1668 | for (int32_t j = 0; j < ubatch.n_seq_id[i]; ++j) { |
| 1669 | llama_seq_id seq_id = ubatch.seq_id[i][j]; |
| 1670 | if (samplers.find(seq_id) == samplers.end()) { |
| 1671 | return true; |
| 1672 | } |
| 1673 | } |
| 1674 | } |
| 1675 | return false; // all sequences use backend sampling |
| 1676 | } |
| 1677 | |
| 1678 | int llama_context::decode(const llama_batch & batch_inp) { |
| 1679 | // MTP hook batches carry both token (next-token id) and embd (h_nextn row), |
| 1680 | // so accept either present rather than requiring exactly one. |
| 1681 | GGML_ASSERT(batch_inp.token || batch_inp.embd)if (!(batch_inp.token || batch_inp.embd)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1681, "GGML_ASSERT(%s) failed", "batch_inp.token || batch_inp.embd" ); |
| 1682 | |
| 1683 | if (!memory) { |
| 1684 | LLAMA_LOG_DEBUG("%s: cannot decode batches with this context (calling encode() instead)\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: cannot decode batches with this context (calling encode() instead)\n" , __func__); |
| 1685 | return encode(batch_inp); |
| 1686 | } |
| 1687 | |
| 1688 | if (batch_inp.n_tokens == 0) { |
| 1689 | LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_tokens == 0\n" , __func__); |
| 1690 | return -1; |
| 1691 | } |
| 1692 | |
| 1693 | const auto & vocab = model.vocab; |
| 1694 | const auto & hparams = model.hparams; |
| 1695 | |
| 1696 | const int64_t n_vocab = vocab.n_tokens(); |
| 1697 | const int64_t n_embd = hparams.n_embd_inp(); |
| 1698 | |
| 1699 | // when computing embeddings, all tokens are output |
| 1700 | const bool output_all = cparams.embeddings; |
| 1701 | const bool has_samplers = !sampling.samplers.empty(); |
| 1702 | |
| 1703 | const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max; |
| 1704 | |
| 1705 | // TODO: avoid this workaround in the future |
| 1706 | if (has_samplers && batch_inp.logits) { |
| 1707 | std::vector<int32_t> seq_output_count(n_seq_max, 0); |
| 1708 | |
| 1709 | for (int32_t i = 0; i < batch_inp.n_tokens; ++i) { |
| 1710 | if (batch_inp.logits[i] == 0) { |
| 1711 | continue; |
| 1712 | } |
| 1713 | |
| 1714 | const int ns = batch_inp.n_seq_id ? batch_inp.n_seq_id[i] : 1; |
| 1715 | |
| 1716 | for (int32_t s = 0; s < ns; ++s) { |
| 1717 | const llama_seq_id seq_id = batch_inp.seq_id ? batch_inp.seq_id[i][s] : 0; |
| 1718 | |
| 1719 | seq_output_count[seq_id]++; |
| 1720 | if (seq_output_count[seq_id] > 1) { |
| 1721 | LLAMA_LOG_ERROR("%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n" , __func__, seq_id, seq_output_count[seq_id]) |
| 1722 | __func__, seq_id, seq_output_count[seq_id])llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n" , __func__, seq_id, seq_output_count[seq_id]); |
| 1723 | return -1; |
| 1724 | } |
| 1725 | } |
| 1726 | } |
| 1727 | } |
| 1728 | |
| 1729 | if (!balloc->init(batch_inp, vocab, memory.get(), n_embd, n_seq_max, output_all)) { |
| 1730 | LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__); |
| 1731 | return -1; |
| 1732 | } |
| 1733 | |
| 1734 | const uint32_t n_tokens_all = balloc->get_n_tokens(); |
| 1735 | const uint32_t n_outputs_all = balloc->get_n_outputs(); |
| 1736 | |
| 1737 | if (output_all) { |
| 1738 | // require that all tokens are output |
| 1739 | if (n_outputs_all != n_tokens_all) { |
| 1740 | LLAMA_LOG_ERROR("%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n" , __func__, n_outputs_all, n_tokens_all) |
| 1741 | __func__, n_outputs_all, n_tokens_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n" , __func__, n_outputs_all, n_tokens_all); |
| 1742 | return -1; |
| 1743 | } |
| 1744 | } |
| 1745 | |
| 1746 | GGML_ASSERT(n_tokens_all <= cparams.n_batch)if (!(n_tokens_all <= cparams.n_batch)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1746, "GGML_ASSERT(%s) failed", "n_tokens_all <= cparams.n_batch" ); |
| 1747 | |
| 1748 | GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens")if (!((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all ) && "non-causal attention requires n_ubatch >= n_tokens" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1748, "GGML_ASSERT(%s) failed", "(cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && \"non-causal attention requires n_ubatch >= n_tokens\"" ); |
| 1749 | |
| 1750 | if (t_compute_start_us == 0) { |
| 1751 | t_compute_start_us = ggml_time_us(); |
| 1752 | } |
| 1753 | n_queued_tokens += n_tokens_all; |
| 1754 | |
| 1755 | // TODO: this clear of the buffer can easily be forgotten - need something better |
| 1756 | embd_seq.clear(); |
| 1757 | output_swaps.clear(); |
| 1758 | |
| 1759 | sched_reserve(); |
| 1760 | |
| 1761 | bool did_optimize = false; |
| 1762 | |
| 1763 | // handle any pending shifts/copies |
| 1764 | memory_update(false); |
| 1765 | |
| 1766 | llama_memory_context_ptr mctx; |
| 1767 | |
| 1768 | while (true) { |
| 1769 | mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all); |
| 1770 | if (!mctx) { |
| 1771 | return -2; |
| 1772 | } |
| 1773 | |
| 1774 | switch (mctx->get_status()) { |
| 1775 | case LLAMA_MEMORY_STATUS_SUCCESS: |
| 1776 | { |
| 1777 | } break; |
| 1778 | case LLAMA_MEMORY_STATUS_NO_UPDATE: |
| 1779 | { |
| 1780 | LLAMA_LOG_ERROR("%s: unexpected memory context status: %d\n", __func__, mctx->get_status())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unexpected memory context status: %d\n" , __func__, mctx->get_status()); |
| 1781 | |
| 1782 | return -2; |
| 1783 | } |
| 1784 | case LLAMA_MEMORY_STATUS_FAILED_PREPARE: |
| 1785 | { |
| 1786 | if (!did_optimize) { |
| 1787 | did_optimize = true; |
| 1788 | |
| 1789 | if (memory_update(true)) { |
| 1790 | LLAMA_LOG_DEBUG("%s: retrying batch size %d after cache optimization\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: retrying batch size %d after cache optimization\n" , __func__, balloc->get_n_tokens()); |
| 1791 | |
| 1792 | continue; |
| 1793 | } |
| 1794 | } |
| 1795 | |
| 1796 | LLAMA_LOG_WARN("%s: failed to find a memory slot for batch of size %d\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: failed to find a memory slot for batch of size %d\n" , __func__, balloc->get_n_tokens()); |
| 1797 | |
| 1798 | return 1; |
| 1799 | } |
| 1800 | case LLAMA_MEMORY_STATUS_FAILED_COMPUTE: |
| 1801 | { |
| 1802 | LLAMA_LOG_ERROR("%s: compute failed while preparing batch of size %d\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: compute failed while preparing batch of size %d\n" , __func__, balloc->get_n_tokens()); |
| 1803 | |
| 1804 | return -2; |
| 1805 | } |
| 1806 | } |
| 1807 | |
| 1808 | break; |
| 1809 | } |
| 1810 | |
| 1811 | // reserve output buffer |
| 1812 | if (output_reserve(n_outputs_all) < n_outputs_all) { |
| 1813 | LLAMA_LOG_ERROR("%s: could not reserve space for batch with %d outputs\n", __func__, n_outputs_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %d outputs\n" , __func__, n_outputs_all); |
| 1814 | return -2; |
| 1815 | }; |
| 1816 | |
| 1817 | int64_t n_outputs_prev = 0; |
| 1818 | int64_t n_tokens_prev = 0; |
| 1819 | |
| 1820 | do { |
| 1821 | const auto & ubatch = mctx->get_ubatch(); |
| 1822 | |
| 1823 | // count the outputs in this ubatch |
| 1824 | { |
| 1825 | int32_t n_outputs_new = 0; |
| 1826 | |
| 1827 | if (n_outputs_all == n_tokens_all) { |
| 1828 | n_outputs_new = ubatch.n_tokens; |
| 1829 | } else { |
| 1830 | for (uint32_t i = 0; i < ubatch.n_tokens; i++) { |
| 1831 | n_outputs_new += (int32_t) (ubatch.output[i] != 0); |
| 1832 | } |
| 1833 | } |
| 1834 | |
| 1835 | // needs to happen before the graph is built |
| 1836 | n_outputs = n_outputs_new; |
| 1837 | } |
| 1838 | |
| 1839 | ggml_status status; |
| 1840 | |
| 1841 | const auto * res = process_ubatch(ubatch, ctx_type_to_graph_type(cparams.ctx_type), mctx.get(), status); |
| 1842 | |
| 1843 | if (!res) { |
| 1844 | // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module |
| 1845 | llama_pos pos_min[LLAMA_MAX_SEQ256]; |
| 1846 | for (int s = 0; s < LLAMA_MAX_SEQ256; ++s) { |
| 1847 | pos_min[s] = std::numeric_limits<llama_pos>::max(); |
| 1848 | } |
| 1849 | |
| 1850 | for (uint32_t i = 0; i < ubatch.n_tokens; ++i) { |
| 1851 | const auto & seq_id = ubatch.seq_id[i][0]; |
| 1852 | |
| 1853 | pos_min[seq_id] = std::min(pos_min[seq_id], ubatch.pos[i]); |
| 1854 | } |
| 1855 | |
| 1856 | for (int s = 0; s < LLAMA_MAX_SEQ256; ++s) { |
| 1857 | if (pos_min[s] == std::numeric_limits<llama_pos>::max()) { |
| 1858 | continue; |
| 1859 | } |
| 1860 | |
| 1861 | LLAMA_LOG_WARN("%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\n", __func__, s, pos_min[s])llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\n" , __func__, s, pos_min[s]); |
| 1862 | |
| 1863 | memory->seq_rm(s, pos_min[s], -1); |
| 1864 | } |
| 1865 | |
| 1866 | switch (status) { |
| 1867 | case GGML_STATUS_ABORTED: return 2; |
| 1868 | case GGML_STATUS_ALLOC_FAILED: return -2; |
| 1869 | case GGML_STATUS_FAILED: return -3; |
| 1870 | case GGML_STATUS_SUCCESS: GGML_ABORT("should not happen")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1870, "should not happen"); |
| 1871 | } |
| 1872 | } |
| 1873 | |
| 1874 | // plot the computation graph in dot format (for debugging purposes) |
| 1875 | //if (n_past%100 == 0) { |
| 1876 | // ggml_graph_dump_dot(gf, NULL, "llama.dot"); |
| 1877 | //} |
| 1878 | |
| 1879 | auto * t_logits = res->get_logits(); |
| 1880 | auto * t_embd = cparams.embeddings ? res->get_embd() : nullptr; |
| 1881 | auto * t_h_nextn = cparams.embeddings_nextn ? res->get_h_nextn() : nullptr; |
| 1882 | |
| 1883 | if (t_embd && res->get_embd_pooled()) { |
| 1884 | t_embd = res->get_embd_pooled(); |
| 1885 | } |
| 1886 | |
| 1887 | // extract logits |
| 1888 | if (logits.data && t_logits && n_outputs > 0 && needs_raw_logits(ubatch, sampling.samplers)) { |
| 1889 | ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits); |
| 1890 | GGML_ASSERT(backend_res != nullptr)if (!(backend_res != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1890, "GGML_ASSERT(%s) failed", "backend_res != nullptr"); |
| 1891 | GGML_ASSERT(logits.data != nullptr)if (!(logits.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1891, "GGML_ASSERT(%s) failed", "logits.data != nullptr"); |
| 1892 | |
| 1893 | float * logits_out = logits.data + n_outputs_prev*n_vocab; |
| 1894 | |
| 1895 | if (n_outputs) { |
| 1896 | GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all)if (!(n_outputs_prev + n_outputs <= n_outputs_all)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1896, "GGML_ASSERT(%s) failed", "n_outputs_prev + n_outputs <= n_outputs_all" ); |
| 1897 | GGML_ASSERT((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits.size)if (!((n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits .size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1897, "GGML_ASSERT(%s) failed", "(n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits.size" ); |
| 1898 | ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputs*n_vocab*sizeof(float)); |
| 1899 | } |
| 1900 | } |
| 1901 | |
| 1902 | // extract embeddings |
| 1903 | if (embd.data && t_embd && n_outputs > 0) { |
| 1904 | ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd); |
| 1905 | GGML_ASSERT(backend_embd != nullptr)if (!(backend_embd != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1905, "GGML_ASSERT(%s) failed", "backend_embd != nullptr"); |
| 1906 | |
| 1907 | switch (cparams.pooling_type) { |
| 1908 | case LLAMA_POOLING_TYPE_NONE: |
| 1909 | { |
| 1910 | // extract token embeddings |
| 1911 | GGML_ASSERT(embd.data != nullptr)if (!(embd.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1911, "GGML_ASSERT(%s) failed", "embd.data != nullptr"); |
| 1912 | const uint32_t n_embd_out = hparams.n_embd_out(); |
| 1913 | float * embd_out = embd.data + n_outputs_prev*n_embd_out; |
| 1914 | |
| 1915 | if (n_outputs) { |
| 1916 | GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all)if (!(n_outputs_prev + n_outputs <= n_outputs_all)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1916, "GGML_ASSERT(%s) failed", "n_outputs_prev + n_outputs <= n_outputs_all" ); |
| 1917 | GGML_ASSERT((n_outputs_prev + n_outputs)*n_embd_out <= (int64_t) embd.size)if (!((n_outputs_prev + n_outputs)*n_embd_out <= (int64_t) embd.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1917, "GGML_ASSERT(%s) failed", "(n_outputs_prev + n_outputs)*n_embd_out <= (int64_t) embd.size" ); |
| 1918 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputs*n_embd_out*sizeof(float)); |
| 1919 | } |
| 1920 | } break; |
| 1921 | case LLAMA_POOLING_TYPE_MEAN: |
| 1922 | case LLAMA_POOLING_TYPE_CLS: |
| 1923 | case LLAMA_POOLING_TYPE_LAST: |
| 1924 | { |
| 1925 | // extract sequence embeddings (cleared before processing each batch) |
| 1926 | auto & embd_seq_out = embd_seq; |
| 1927 | |
| 1928 | // use n_embd_out (not n_embd_inp) - the pooled embedding has the model's |
| 1929 | // output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl) |
| 1930 | const uint32_t n_embd_out = hparams.n_embd_out(); |
| 1931 | |
| 1932 | for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { |
| 1933 | const llama_seq_id seq_id = ubatch.seq_id_unq[s]; |
| 1934 | const int32_t seq_idx = ubatch.seq_idx[seq_id]; |
| 1935 | |
| 1936 | embd_seq_out[seq_id].resize(n_embd_out); |
| 1937 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_out*seq_idx)*sizeof(float), n_embd_out*sizeof(float)); |
| 1938 | } |
| 1939 | } break; |
| 1940 | case LLAMA_POOLING_TYPE_RANK: |
| 1941 | { |
| 1942 | // extract the rerank score - n_cls_out floats per sequence |
| 1943 | auto & embd_seq_out = embd_seq; |
| 1944 | |
| 1945 | const uint32_t n_cls_out = hparams.n_cls_out; |
| 1946 | |
| 1947 | for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) { |
| 1948 | const llama_seq_id seq_id = ubatch.seq_id_unq[s]; |
| 1949 | const int32_t seq_idx = ubatch.seq_idx[seq_id]; |
| 1950 | |
| 1951 | embd_seq_out[seq_id].resize(n_cls_out); |
| 1952 | ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_out*seq_idx)*sizeof(float), n_cls_out*sizeof(float)); |
| 1953 | } |
| 1954 | } break; |
| 1955 | case LLAMA_POOLING_TYPE_UNSPECIFIED: |
| 1956 | { |
| 1957 | GGML_ABORT("unknown pooling type")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1957, "unknown pooling type"); |
| 1958 | } |
| 1959 | } |
| 1960 | } |
| 1961 | |
| 1962 | extract_layer_inputs(res, n_tokens_prev, ubatch.n_tokens); |
| 1963 | |
| 1964 | // extract nextn embeddings before |
| 1965 | // only meaningful in LLAMA_POOLING_TYPE_NONE (per-token); other pooling modes are ignored. |
| 1966 | { |
| 1967 | const bool masked = cparams.embeddings_nextn_masked; |
| 1968 | const int64_t n_rows = masked ? n_outputs : (int64_t) ubatch.n_tokens; |
| 1969 | const int64_t offset = masked ? n_outputs_prev : n_tokens_prev; |
| 1970 | |
| 1971 | if (embd_nextn.data && t_h_nextn && n_rows > 0 && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) { |
| 1972 | ggml_backend_t backend_h = ggml_backend_sched_get_tensor_backend(sched.get(), t_h_nextn); |
| 1973 | GGML_ASSERT(backend_h != nullptr)if (!(backend_h != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1973, "GGML_ASSERT(%s) failed", "backend_h != nullptr"); |
| 1974 | |
| 1975 | const uint32_t n_embd = hparams.n_embd_out(); |
| 1976 | float * embd_nextn_out = embd_nextn.data + offset*n_embd; |
| 1977 | |
| 1978 | GGML_ASSERT((offset + n_rows)*n_embd <= (int64_t) embd_nextn.size)if (!((offset + n_rows)*n_embd <= (int64_t) embd_nextn.size )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1978, "GGML_ASSERT(%s) failed", "(offset + n_rows)*n_embd <= (int64_t) embd_nextn.size" ); |
| 1979 | ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn_out, 0, n_rows*n_embd*sizeof(float)); |
| 1980 | } |
| 1981 | } |
| 1982 | |
| 1983 | // Copy backend sampling output if this ubatch produced any sampling tensors. |
| 1984 | if (has_samplers && (!res->t_sampled.empty() || !res->t_sampled_probs.empty() || !res->t_sampled_logits.empty())) { |
| 1985 | const auto seq_to_output_row = build_seq_to_output_row(ubatch, n_outputs_prev); |
| 1986 | const auto stride = n_vocab; |
| 1987 | |
| 1988 | // async copy the sampling data from the backend to the host |
| 1989 | copy_tensor_async_ints(res->t_sampled, sampling.sampled, seq_to_output_row, sched.get()); |
| 1990 | |
| 1991 | copy_tensor_async_floats (res->t_sampled_logits, sampling.logits, stride, sampling.logits_count, seq_to_output_row, sched.get()); |
| 1992 | copy_tensor_async_floats (res->t_sampled_probs, sampling.probs, stride, sampling.probs_count, seq_to_output_row, sched.get()); |
| 1993 | copy_tensor_async_candidates(res->t_candidates, sampling.candidates, stride, sampling.candidates_count, seq_to_output_row, sched.get()); |
| 1994 | } |
| 1995 | |
| 1996 | n_outputs_prev += n_outputs; |
| 1997 | n_tokens_prev += ubatch.n_tokens; |
| 1998 | } while (mctx->next()); |
| 1999 | |
| 2000 | // set to total number of outputs in the batch, for use in llama_get_logits_ith |
| 2001 | n_outputs = n_outputs_all; |
| 2002 | |
| 2003 | // set output mappings |
| 2004 | if (n_outputs > 0) { |
| 2005 | bool sorted_output = true; |
| 2006 | |
| 2007 | auto & out_ids = balloc->get_out_ids(); |
| 2008 | |
| 2009 | GGML_ASSERT(out_ids.size() == (size_t) n_outputs)if (!(out_ids.size() == (size_t) n_outputs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2009, "GGML_ASSERT(%s) failed", "out_ids.size() == (size_t) n_outputs" ); |
| 2010 | |
| 2011 | for (int64_t i = 0; i < n_outputs; ++i) { |
| 2012 | int64_t out_id = out_ids[i]; |
| 2013 | output_ids[out_id] = i; |
| 2014 | if (out_id != i) { |
| 2015 | sorted_output = false; |
| 2016 | } |
| 2017 | } |
| 2018 | |
| 2019 | // make the outputs have the same order they had in the user-provided batch |
| 2020 | // note: this is mostly relevant for recurrent models atm |
| 2021 | if (!sorted_output && n_outputs > 1) { |
| 2022 | GGML_ASSERT((size_t) n_outputs == out_ids.size())if (!((size_t) n_outputs == out_ids.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2022, "GGML_ASSERT(%s) failed", "(size_t) n_outputs == out_ids.size()" ); |
| 2023 | |
| 2024 | // TODO: is there something more efficient which also minimizes swaps? |
| 2025 | // selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort) |
| 2026 | for (uint32_t i = 0; i < n_outputs - 1; ++i) { |
| 2027 | uint32_t j_min = i; |
| 2028 | for (uint32_t j = i + 1; j < n_outputs; ++j) { |
| 2029 | if (out_ids[j] < out_ids[j_min]) { |
| 2030 | j_min = j; |
| 2031 | } |
| 2032 | } |
| 2033 | if (j_min == i) { |
| 2034 | continue; |
| 2035 | } |
| 2036 | std::swap(out_ids[i], out_ids[j_min]); |
| 2037 | |
| 2038 | // remember the swaps and apply them lazily upon logits/embeddings access |
| 2039 | output_swaps.push_back({ i, j_min }); |
| 2040 | } |
| 2041 | |
| 2042 | std::fill(output_ids.begin(), output_ids.end(), -1); |
| 2043 | |
| 2044 | for (uint32_t i = 0; i < n_outputs; ++i) { |
| 2045 | output_ids[out_ids[i]] = i; |
| 2046 | } |
| 2047 | } |
| 2048 | } |
| 2049 | |
| 2050 | // wait for the computation to finish (automatically done when obtaining the model output) |
| 2051 | //synchronize(); |
| 2052 | |
| 2053 | return 0; |
| 2054 | } |
| 2055 | |
| 2056 | // |
| 2057 | // output |
| 2058 | // |
| 2059 | |
| 2060 | uint32_t llama_context::output_reserve(int32_t n_outputs) { |
| 2061 | const auto & hparams = model.hparams; |
| 2062 | const auto & vocab = model.vocab; |
| 2063 | |
| 2064 | const int64_t n_outputs_max = std::max<int64_t>(n_outputs, n_seq_max()); |
| 2065 | |
| 2066 | const auto n_batch = cparams.n_batch; |
| 2067 | const auto n_vocab = vocab.n_tokens(); |
| 2068 | const auto n_embd = hparams.n_embd; |
| 2069 | const auto n_embd_out = hparams.n_embd_out(); |
| 2070 | |
| 2071 | bool has_logits = true; |
| 2072 | bool has_embd = cparams.embeddings; |
| 2073 | bool has_embd_nextn = cparams.embeddings_nextn; |
| 2074 | |
| 2075 | // TODO: hacky enc-dec support |
| 2076 | if (model.arch == LLM_ARCH_T5) { |
| 2077 | has_logits = true; |
| 2078 | has_embd = true; |
| 2079 | } |
| 2080 | |
| 2081 | size_t backend_float_count = 0; |
| 2082 | size_t backend_token_count = 0; |
| 2083 | size_t embd_layer_inp_float_count = 0; |
| 2084 | |
| 2085 | logits.size = has_logits ? n_vocab*n_outputs_max : 0; |
| 2086 | embd.size = has_embd ? n_embd_out*n_outputs_max : 0; |
| 2087 | embd_nextn.size = has_embd_nextn ? n_embd_out*n_outputs_max : 0; |
| 2088 | |
| 2089 | if (has_embd_nextn && !cparams.embeddings_nextn_masked) { |
| 2090 | // unmasked: nextn row exists for every token in the batch, not just |
| 2091 | // those flagged via batch.logits[i] -> size by token count instead. |
| 2092 | embd_nextn.size = (size_t) n_embd_out * n_batch; |
| 2093 | } |
| 2094 | |
| 2095 | for (bool enabled : cparams.embeddings_layer_inp) { |
| 2096 | if (enabled) { |
| 2097 | embd_layer_inp_float_count += (size_t) n_embd * n_batch; |
| 2098 | } |
| 2099 | } |
| 2100 | |
| 2101 | // Allocate backend sampling output buffers if there are backend samplers configured. |
| 2102 | const bool has_sampling = !sampling.samplers.empty(); |
| 2103 | if (has_sampling) { |
| 2104 | backend_float_count = 2 * n_vocab * n_outputs_max; // logits + probs |
| 2105 | backend_token_count = (1 + n_vocab) * n_outputs_max; // sampled + candidates |
| 2106 | } |
| 2107 | |
| 2108 | if (output_ids.empty()) { |
| 2109 | // init, never resized afterwards |
| 2110 | output_ids.resize(n_batch); |
| 2111 | } |
| 2112 | |
| 2113 | const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0; |
| 2114 | const size_t new_size = |
| 2115 | (logits.size + embd.size + embd_nextn.size + embd_layer_inp_float_count + backend_float_count) * sizeof(float) + |
| 2116 | ( backend_token_count) * sizeof(llama_token); |
| 2117 | |
| 2118 | // alloc only when more than the current capacity is required |
| 2119 | // TODO: also consider shrinking the buffer |
| 2120 | if (!buf_output || prev_size < new_size) { |
| 2121 | if (buf_output) { |
| 2122 | #ifndef NDEBUG |
| 2123 | // This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark) |
| 2124 | LLAMA_LOG_DEBUG("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n" , __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0 ); |
| 2125 | #endif |
| 2126 | synchronize(); |
| 2127 | |
| 2128 | // TODO: not needed? |
| 2129 | buf_output = nullptr; |
| 2130 | logits.data = nullptr; |
| 2131 | embd.data = nullptr; |
| 2132 | embd_nextn.data = nullptr; |
| 2133 | for (auto & layer_inp : embd_layer_inp) { |
| 2134 | layer_inp = {nullptr, 0}; |
| 2135 | } |
| 2136 | } |
| 2137 | |
| 2138 | auto * buft = ggml_backend_cpu_buffer_type(); |
| 2139 | // try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory |
| 2140 | auto * output_dev = model.dev_output(); |
| 2141 | auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr; |
| 2142 | if (output_dev_host_buft) { |
| 2143 | buft = output_dev_host_buft; |
| 2144 | } |
| 2145 | buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size)); |
| 2146 | if (buf_output == nullptr) { |
| 2147 | LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate output buffer of size %.2f MiB\n" , __func__, new_size / (1024.0 * 1024.0)); |
| 2148 | return 0; |
| 2149 | } |
| 2150 | ggml_backend_buffer_clear(buf_output.get(), 0); |
| 2151 | } |
| 2152 | |
| 2153 | float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get()); |
| 2154 | |
| 2155 | size_t offset = 0; |
| 2156 | uint8_t * base = (uint8_t *) output_base; |
| 2157 | |
| 2158 | logits = has_logits ? buffer_view<float>{output_base, logits.size} : buffer_view<float>{nullptr, 0}; |
| 2159 | offset += logits.size * sizeof(float); |
| 2160 | |
| 2161 | embd = has_embd ? buffer_view<float>{(float *) (base + offset), embd.size} : buffer_view<float>{nullptr, 0}; |
| 2162 | offset += embd.size * sizeof(float); |
| 2163 | |
| 2164 | embd_nextn = has_embd_nextn ? buffer_view<float>{(float *) (base + offset), embd_nextn.size} : buffer_view<float>{nullptr, 0}; |
| 2165 | offset += embd_nextn.size * sizeof(float); |
| 2166 | |
| 2167 | for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) { |
| 2168 | if (cparams.embeddings_layer_inp[il]) { |
| 2169 | embd_layer_inp[il] = buffer_view<float>{(float *) (base + offset), (size_t) n_embd * n_batch}; |
| 2170 | offset += embd_layer_inp[il].size * sizeof(float); |
| 2171 | } else { |
| 2172 | embd_layer_inp[il] = buffer_view<float>{nullptr, 0}; |
| 2173 | } |
| 2174 | } |
| 2175 | |
| 2176 | if (has_sampling) { |
| 2177 | sampling.logits = {(float *) (base + offset), (size_t)(n_vocab*n_outputs_max)}; |
| 2178 | offset += sampling.logits.size * sizeof(float); |
| 2179 | |
| 2180 | sampling.probs = {(float *) (base + offset), (size_t)(n_vocab*n_outputs_max)}; |
| 2181 | offset += sampling.probs.size * sizeof(float); |
| 2182 | |
| 2183 | sampling.sampled = {(llama_token *) (base + offset), (size_t)n_outputs_max}; |
| 2184 | offset += sampling.sampled.size * sizeof(llama_token); |
| 2185 | |
| 2186 | sampling.candidates = {(llama_token *) (base + offset), (size_t)(n_vocab*n_outputs_max)}; |
| 2187 | offset += sampling.candidates.size * sizeof(llama_token); |
| 2188 | |
| 2189 | // The count vectors keep track of the actual number of logits/probs/candidates |
| 2190 | // copied from the backend for each output row. |
| 2191 | |
| 2192 | sampling.logits_count.resize(n_outputs_max); |
| 2193 | sampling.probs_count.resize(n_outputs_max); |
| 2194 | sampling.candidates_count.resize(n_outputs_max); |
| 2195 | |
| 2196 | std::fill(sampling.logits_count.begin(), sampling.logits_count.end(), 0); |
| 2197 | std::fill(sampling.probs_count.begin(), sampling.probs_count.end(), 0); |
| 2198 | std::fill(sampling.candidates_count.begin(), sampling.candidates_count.end(), 0); |
| 2199 | |
| 2200 | std::fill_n(sampling.sampled.data, sampling.sampled.size, LLAMA_TOKEN_NULL-1); |
| 2201 | } else { |
| 2202 | sampling.logits = {nullptr, 0}; |
| 2203 | sampling.probs = {nullptr, 0}; |
| 2204 | sampling.sampled = {nullptr, 0}; |
| 2205 | sampling.candidates = {nullptr, 0}; |
| 2206 | |
| 2207 | sampling.logits_count.clear(); |
| 2208 | sampling.probs_count.clear(); |
| 2209 | sampling.candidates_count.clear(); |
| 2210 | } |
| 2211 | |
| 2212 | // set all ids as invalid (negative) |
| 2213 | std::fill(output_ids.begin(), output_ids.end(), -1); |
| 2214 | |
| 2215 | this->n_outputs = 0; |
| 2216 | |
| 2217 | GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max)if (!(n_outputs_max <= cparams.n_outputs_max)) ggml_abort( "/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2217, "GGML_ASSERT(%s) failed", "n_outputs_max <= cparams.n_outputs_max" ); |
| 2218 | |
| 2219 | return n_outputs_max; |
| 2220 | } |
| 2221 | |
| 2222 | void llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) { |
| 2223 | for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) { |
| 2224 | if (!cparams.embeddings_layer_inp[il]) { |
| 2225 | continue; |
| 2226 | } |
| 2227 | if (!embd_layer_inp[il].has_data()) { |
| 2228 | GGML_ABORT("output layer input buffer not allocated")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2228, "output layer input buffer not allocated"); |
| 2229 | } |
| 2230 | ggml_tensor * t = res->get_layer_inp((int) il); |
| 2231 | if (!t) { |
| 2232 | GGML_ABORT("layer input tensor not found")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2232, "layer input tensor not found"); |
| 2233 | } |
| 2234 | |
| 2235 | const size_t nbytes = ggml_nbytes(t); |
| 2236 | const size_t nfloats = nbytes / sizeof(float); |
| 2237 | GGML_ASSERT(n_tokens > 0)if (!(n_tokens > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2237, "GGML_ASSERT(%s) failed", "n_tokens > 0"); |
| 2238 | GGML_ASSERT(nfloats % n_tokens == 0)if (!(nfloats % n_tokens == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2238, "GGML_ASSERT(%s) failed", "nfloats % n_tokens == 0"); |
| 2239 | |
| 2240 | const size_t row_floats = nfloats / n_tokens; |
| 2241 | const size_t dst_offset = token_offset * row_floats; |
| 2242 | GGML_ASSERT(dst_offset + nfloats <= embd_layer_inp[il].size)if (!(dst_offset + nfloats <= embd_layer_inp[il].size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2242, "GGML_ASSERT(%s) failed", "dst_offset + nfloats <= embd_layer_inp[il].size" ); |
| 2243 | |
| 2244 | ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched.get(), t); |
| 2245 | GGML_ASSERT(backend != nullptr)if (!(backend != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2245, "GGML_ASSERT(%s) failed", "backend != nullptr"); |
| 2246 | ggml_backend_tensor_get_async(backend, t, embd_layer_inp[il].data + dst_offset, 0, nbytes); |
| 2247 | } |
| 2248 | } |
| 2249 | |
| 2250 | void llama_context::output_reorder() { |
| 2251 | const uint64_t n_vocab = model.vocab.n_tokens(); |
| 2252 | const uint64_t n_embd = model.hparams.n_embd; |
| 2253 | |
| 2254 | for (size_t s = 0; s < output_swaps.size(); ++s) { |
| 2255 | const uint64_t i0 = output_swaps[s].i0; |
| 2256 | const uint64_t i1 = output_swaps[s].i1; |
| 2257 | |
| 2258 | if (logits.size > 0) { |
| 2259 | for (uint64_t k = 0; k < n_vocab; k++) { |
| 2260 | std::swap(logits.data[i0*n_vocab + k], logits.data[i1*n_vocab + k]); |
| 2261 | } |
| 2262 | } |
| 2263 | |
| 2264 | if (embd.size > 0) { |
| 2265 | for (uint64_t k = 0; k < n_embd; k++) { |
| 2266 | std::swap(embd.data[i0*n_embd + k], embd.data[i1*n_embd + k]); |
| 2267 | } |
| 2268 | } |
| 2269 | |
| 2270 | if (embd_nextn.size > 0) { |
| 2271 | for (uint64_t k = 0; k < n_embd; k++) { |
| 2272 | std::swap(embd_nextn.data[i0*n_embd + k], embd_nextn.data[i1*n_embd + k]); |
| 2273 | } |
| 2274 | } |
| 2275 | |
| 2276 | if (embd_layer_inp.size() > 0) { |
| 2277 | for (int lid = 0; lid < (int) embd_layer_inp.size(); ++lid) { |
| 2278 | if (embd_layer_inp[lid].size > 0) { |
| 2279 | for (uint64_t k = 0; k < n_embd; ++k) { |
| 2280 | std::swap(embd_layer_inp[lid].data[i0*n_embd + k], embd_layer_inp[lid].data[i1*n_embd + k]); |
| 2281 | } |
| 2282 | } |
| 2283 | } |
| 2284 | } |
| 2285 | |
| 2286 | if (!sampling.samplers.empty()) { |
| 2287 | assert(sampling.logits.size > 0)(static_cast <bool> (sampling.logits.size > 0) ? void (0) : __assert_fail ("sampling.logits.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2288 | assert(sampling.probs.size > 0)(static_cast <bool> (sampling.probs.size > 0) ? void (0) : __assert_fail ("sampling.probs.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2289 | assert(sampling.candidates.size > 0)(static_cast <bool> (sampling.candidates.size > 0) ? void (0) : __assert_fail ("sampling.candidates.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ )); |
| 2290 | assert(sampling.sampled.size > 0)(static_cast <bool> (sampling.sampled.size > 0) ? void (0) : __assert_fail ("sampling.sampled.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__)); |
| 2291 | assert(sampling.logits_count.size() > 0)(static_cast <bool> (sampling.logits_count.size() > 0 ) ? void (0) : __assert_fail ("sampling.logits_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ )); |
| 2292 | assert(sampling.probs_count.size() > 0)(static_cast <bool> (sampling.probs_count.size() > 0 ) ? void (0) : __assert_fail ("sampling.probs_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ )); |
| 2293 | assert(sampling.candidates_count.size() > 0)(static_cast <bool> (sampling.candidates_count.size() > 0) ? void (0) : __assert_fail ("sampling.candidates_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ )); |
| 2294 | |
| 2295 | for (uint64_t k = 0; k < n_vocab; ++k) { |
| 2296 | std::swap(sampling.logits.data[i0*n_vocab + k], sampling.logits.data[i1*n_vocab + k]); |
| 2297 | } |
| 2298 | |
| 2299 | for (uint64_t k = 0; k < n_vocab; ++k) { |
| 2300 | std::swap(sampling.probs.data[i0*n_vocab + k], sampling.probs.data[i1*n_vocab + k]); |
| 2301 | } |
| 2302 | |
| 2303 | for (uint64_t k = 0; k < n_vocab; ++k) { |
| 2304 | std::swap(sampling.candidates.data[i0*n_vocab + k], sampling.candidates.data[i1*n_vocab + k]); |
| 2305 | } |
| 2306 | |
| 2307 | std::swap(sampling.sampled.data[i0], sampling.sampled.data[i1]); |
| 2308 | std::swap(sampling.logits_count[i0], sampling.logits_count[i1]); |
| 2309 | std::swap(sampling.probs_count[i0], sampling.probs_count[i1]); |
| 2310 | std::swap(sampling.candidates_count[i0], sampling.candidates_count[i1]); |
| 2311 | } |
| 2312 | } |
| 2313 | |
| 2314 | output_swaps.clear(); |
| 2315 | } |
| 2316 | |
| 2317 | // |
| 2318 | // graph |
| 2319 | // |
| 2320 | |
| 2321 | uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { |
| 2322 | if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE) { |
| 2323 | return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors()); |
| 2324 | } |
| 2325 | uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors()); |
| 2326 | for (const auto & lora : model.loras) { |
| 2327 | res += lora->get_n_nodes(); |
| 2328 | } |
| 2329 | return res; |
| 2330 | } |
| 2331 | |
| 2332 | llm_graph_result * llama_context::get_gf_res_reserve() const { |
| 2333 | return static_cast<llm_graph_result *>(gf_res_reserve.get()); |
| 2334 | } |
| 2335 | |
| 2336 | ggml_cgraph * llama_context::graph_reserve( |
| 2337 | uint32_t n_tokens, uint32_t n_seqs, uint32_t n_outputs, const llama_memory_context_i * mctx, bool split_only, size_t * sizes) { |
| 2338 | LLAMA_LOG_DEBUG("%s: reserving a graph for ubatch with n_tokens = %4u, n_seqs = %2u, n_outputs = %4u\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reserving a graph for ubatch with n_tokens = %4u, n_seqs = %2u, n_outputs = %4u\n" , __func__, n_tokens, n_seqs, n_outputs); |
| 2339 | GGML_ASSERT(n_outputs >= 1)if (!(n_outputs >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2339, "GGML_ASSERT(%s) failed", "n_outputs >= 1"); |
| 2340 | |
| 2341 | if (n_tokens % n_seqs != 0) { |
| 2342 | n_tokens = ((n_tokens + (n_seqs - 1)) / n_seqs) * n_seqs; // round to next multiple of n_seqs |
| 2343 | LLAMA_LOG_DEBUG("%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n" , __func__, n_tokens, n_seqs, n_outputs); |
| 2344 | } |
| 2345 | |
| 2346 | ggml_backend_sched_reset(sched.get()); |
| 2347 | |
| 2348 | // when the scheduler is reset, we cannot reuse the old graph, so we reset the previous graph result to prevent that |
| 2349 | gf_res_prev->reset(); |
| 2350 | |
| 2351 | // store the n_outputs as it is, and restore it afterwards |
| 2352 | // TODO: not sure if needed, might simplify in the future by removing this |
| 2353 | const auto save_n_outputs = this->n_outputs; |
| 2354 | |
| 2355 | this->n_outputs = n_outputs; |
| 2356 | |
| 2357 | llama_batch_allocr balloc(model.hparams.n_pos_per_embd()); |
| 2358 | llama_ubatch ubatch = balloc.ubatch_reserve(n_tokens/n_seqs, n_seqs); |
| 2359 | |
| 2360 | // set one output token per sequence in order to activate all backend samplers |
| 2361 | std::vector<llama_seq_id> seq_ids(n_seqs); |
| 2362 | for (uint32_t i = 0; i < n_seqs; ++i) { |
| 2363 | seq_ids[i] = i; |
| 2364 | ubatch.n_seq_id[i] = 1; |
| 2365 | ubatch.seq_id[i] = &seq_ids[i]; |
| 2366 | ubatch.output[i] = true; |
| 2367 | } |
| 2368 | |
| 2369 | auto * res = gf_res_reserve.get(); |
| 2370 | |
| 2371 | const auto gparams = graph_params(res, ubatch, mctx, ctx_type_to_graph_type(cparams.ctx_type)); |
| 2372 | |
| 2373 | res->reset(); |
| 2374 | |
| 2375 | auto * gf = model.build_graph(gparams); |
| 2376 | |
| 2377 | this->n_outputs = save_n_outputs; |
| 2378 | |
| 2379 | // initialize scheduler with the specified graph |
| 2380 | if (split_only) { |
| 2381 | if (sizes) { |
| 2382 | ggml_backend_sched_reserve_size(sched.get(), gf, sizes); |
| 2383 | } else { |
| 2384 | ggml_backend_sched_split_graph(sched.get(), gf); |
| 2385 | } |
| 2386 | } else if (!ggml_backend_sched_reserve(sched.get(), gf)) { |
| 2387 | GGML_ASSERT(!sizes)if (!(!sizes)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2387, "GGML_ASSERT(%s) failed", "!sizes"); |
| 2388 | LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate compute buffers\n" , __func__); |
| 2389 | return nullptr; |
| 2390 | } |
| 2391 | |
| 2392 | return gf; |
| 2393 | } |
| 2394 | |
| 2395 | llm_graph_params llama_context::graph_params( |
| 2396 | llm_graph_result * res, |
| 2397 | const llama_ubatch & ubatch, |
| 2398 | const llama_memory_context_i * mctx, |
| 2399 | llm_graph_type gtype) const { |
| 2400 | return { |
| 2401 | /*.arch =*/ model.arch, |
| 2402 | /*.hparams =*/ model.hparams, |
| 2403 | /*.cparams =*/ cparams, |
| 2404 | /*.ubatch =*/ ubatch, |
| 2405 | /*.gtype =*/ gtype, |
| 2406 | /*.sched =*/ sched.get(), |
| 2407 | /*.backend_cpu =*/ backend_cpu, |
| 2408 | /*.cvec =*/ cvec.get(), |
| 2409 | /*.loras =*/ loras.get(), |
| 2410 | /*.mctx =*/ mctx, |
| 2411 | /*.cross =*/ &cross, |
| 2412 | /*.samplers =*/ sampling.samplers, |
| 2413 | /*.n_outputs =*/ n_outputs, |
| 2414 | /*.cb =*/ graph_get_cb(), |
| 2415 | /*.res =*/ res, |
| 2416 | }; |
| 2417 | } |
| 2418 | |
| 2419 | ggml_status llama_context::graph_compute( |
| 2420 | ggml_cgraph * gf, |
| 2421 | bool batched) { |
| 2422 | int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads; |
| 2423 | ggml_threadpool_t tp = batched ? threadpool_batch : threadpool; |
| 2424 | |
| 2425 | if (backend_cpu != nullptr) { |
| 2426 | auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu)); |
| 2427 | auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool"); |
| 2428 | if (set_threadpool_fn) { |
| 2429 | set_threadpool_fn(backend_cpu, tp); |
| 2430 | } |
| 2431 | } |
| 2432 | |
| 2433 | // set the number of threads for all the backends |
| 2434 | for (const auto & set_n_threads_fn : set_n_threads_fns) { |
| 2435 | set_n_threads_fn.second(set_n_threads_fn.first, n_threads); |
| 2436 | } |
| 2437 | |
| 2438 | auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); |
| 2439 | if (status != GGML_STATUS_SUCCESS) { |
| 2440 | LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: ggml_backend_sched_graph_compute_async failed with error %d\n" , __func__, status); |
| 2441 | } |
| 2442 | |
| 2443 | // fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched)); |
| 2444 | |
| 2445 | return status; |
| 2446 | } |
| 2447 | |
| 2448 | llm_graph_cb llama_context::graph_get_cb() const { |
| 2449 | return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) { |
| 2450 | if (il >= 0) { |
| 2451 | ggml_format_name(cur, "%s-%d", name, il); |
| 2452 | } else { |
| 2453 | ggml_set_name(cur, name); |
| 2454 | } |
| 2455 | |
| 2456 | // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends |
| 2457 | // FIXME: fix in ggml_backend_sched |
| 2458 | const bool full_offload = model.n_gpu_layers() > model.hparams.n_layer_all; |
| 2459 | if (ubatch.n_tokens < 32 || full_offload) { |
| 2460 | if (il != -1 && strcmp(name, "norm") == 0) { |
| 2461 | const auto & dev_layer = model.dev_layer(il); |
| 2462 | for (const auto & backend : backends) { |
| 2463 | if (ggml_backend_get_device(backend.get()) == dev_layer) { |
| 2464 | if (ggml_backend_supports_op(backend.get(), cur)) { |
| 2465 | ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get()); |
| 2466 | } |
| 2467 | } |
| 2468 | } |
| 2469 | } |
| 2470 | } |
| 2471 | }; |
| 2472 | } |
| 2473 | |
| 2474 | // |
| 2475 | // state save/load |
| 2476 | // |
| 2477 | |
| 2478 | class llama_io_write_dummy : public llama_io_write_i { |
| 2479 | public: |
| 2480 | llama_io_write_dummy(bool skip_tensors) : skip_tensors(skip_tensors) {} |
| 2481 | |
| 2482 | void write(const void * /* src */, size_t size) override { |
| 2483 | size_written += size; |
| 2484 | } |
| 2485 | |
| 2486 | void write_tensor(ggml_tensor * /* tensor */, size_t /* offset */, size_t size) override { |
| 2487 | if (skip_tensors) { |
| 2488 | return; |
| 2489 | } |
| 2490 | |
| 2491 | size_written += size; |
| 2492 | } |
| 2493 | |
| 2494 | size_t n_bytes() override { |
| 2495 | return size_written; |
| 2496 | } |
| 2497 | |
| 2498 | private: |
| 2499 | const bool skip_tensors; |
| 2500 | |
| 2501 | size_t size_written = 0; |
| 2502 | }; |
| 2503 | |
| 2504 | class llama_io_write_host : public llama_io_write_i { |
| 2505 | public: |
| 2506 | llama_io_write_host( |
| 2507 | uint8_t * p, size_t len) : ptr(p), buf_size(len) {} |
| 2508 | |
| 2509 | ~llama_io_write_host() { |
| 2510 | // TODO: add backend support to batch tensor_get? or some other way to speed this up |
| 2511 | for (const auto & winfo : winfos) { |
| 2512 | ggml_backend_tensor_get(winfo.tensor, winfo.ptr, winfo.offset, winfo.size); |
| 2513 | } |
| 2514 | } |
| 2515 | |
| 2516 | void write(const void * src, size_t size) override { |
| 2517 | if (size > buf_size) { |
| 2518 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2519 | } |
| 2520 | memcpy(ptr, src, size); |
| 2521 | ptr += size; |
| 2522 | size_written += size; |
| 2523 | buf_size -= size; |
| 2524 | } |
| 2525 | |
| 2526 | void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2527 | if (size > buf_size) { |
| 2528 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2529 | } |
| 2530 | |
| 2531 | // save the write for later during destruction |
| 2532 | winfos.push_back({tensor, ptr, size, offset}); |
| 2533 | |
| 2534 | ptr += size; |
| 2535 | size_written += size; |
| 2536 | buf_size -= size; |
| 2537 | } |
| 2538 | |
| 2539 | size_t n_bytes() override { |
| 2540 | return size_written; |
| 2541 | } |
| 2542 | |
| 2543 | private: |
| 2544 | uint8_t * ptr; |
| 2545 | size_t buf_size = 0; |
| 2546 | size_t size_written = 0; |
| 2547 | |
| 2548 | struct write_info { |
| 2549 | ggml_tensor * tensor; |
| 2550 | uint8_t * ptr; |
| 2551 | size_t size; |
| 2552 | size_t offset; |
| 2553 | }; |
| 2554 | std::vector<write_info> winfos; |
| 2555 | }; |
| 2556 | |
| 2557 | class llama_io_read_host : public llama_io_read_i { |
| 2558 | public: |
| 2559 | llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {} |
| 2560 | |
| 2561 | ~llama_io_read_host() { |
| 2562 | // flush the reads |
| 2563 | for (const auto & rinfo : rinfos) { |
| 2564 | ggml_backend_tensor_set(rinfo.tensor, rinfo.ptr, rinfo.offset, rinfo.size); |
| 2565 | } |
| 2566 | } |
| 2567 | |
| 2568 | void read(void * dst, size_t size) override { |
| 2569 | if (size > buf_size) { |
| 2570 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2571 | } |
| 2572 | memcpy(dst, ptr, size); |
| 2573 | ptr += size; |
| 2574 | size_read += size; |
| 2575 | buf_size -= size; |
| 2576 | } |
| 2577 | |
| 2578 | void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2579 | if (size > buf_size) { |
| 2580 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2581 | } |
| 2582 | |
| 2583 | // save for later during destruction |
| 2584 | rinfos.push_back({tensor, ptr, size, offset}); |
| 2585 | |
| 2586 | ptr += size; |
| 2587 | size_read += size; |
| 2588 | buf_size -= size; |
| 2589 | } |
| 2590 | |
| 2591 | size_t n_bytes() override { |
| 2592 | return size_read; |
| 2593 | } |
| 2594 | |
| 2595 | private: |
| 2596 | const uint8_t * ptr; |
| 2597 | size_t buf_size = 0; |
| 2598 | size_t size_read = 0; |
| 2599 | |
| 2600 | struct read_info { |
| 2601 | ggml_tensor * tensor; |
| 2602 | const uint8_t * ptr; |
| 2603 | size_t size; |
| 2604 | size_t offset; |
| 2605 | }; |
| 2606 | std::vector<read_info> rinfos; |
| 2607 | }; |
| 2608 | |
| 2609 | class llama_io_write_file : public llama_io_write_i { |
| 2610 | public: |
| 2611 | llama_io_write_file(llama_file * f) : file(f) {} |
| 2612 | |
| 2613 | void write(const void * src, size_t size) override { |
| 2614 | file->write_raw(src, size); |
| 2615 | size_written += size; |
| 2616 | } |
| 2617 | |
| 2618 | void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2619 | temp_buffer.resize(size); |
| 2620 | ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size); |
| 2621 | write(temp_buffer.data(), temp_buffer.size()); |
| 2622 | } |
| 2623 | |
| 2624 | size_t n_bytes() override { |
| 2625 | return size_written; |
| 2626 | } |
| 2627 | |
| 2628 | private: |
| 2629 | llama_file * file; |
| 2630 | size_t size_written = 0; |
| 2631 | std::vector<uint8_t> temp_buffer; |
| 2632 | }; |
| 2633 | |
| 2634 | class llama_io_read_file : public llama_io_read_i { |
| 2635 | public: |
| 2636 | llama_io_read_file(llama_file * f) : file(f) {} |
| 2637 | |
| 2638 | void read(void * dst, size_t size) override { |
| 2639 | file->read_raw(dst, size); |
| 2640 | size_read += size; |
| 2641 | } |
| 2642 | |
| 2643 | void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2644 | temp_buffer.resize(size); |
| 2645 | read(temp_buffer.data(), size); |
| 2646 | ggml_backend_tensor_set(tensor, temp_buffer.data(), offset, size); |
| 2647 | } |
| 2648 | |
| 2649 | size_t n_bytes() override { |
| 2650 | return size_read; |
| 2651 | } |
| 2652 | |
| 2653 | private: |
| 2654 | llama_file * file; |
| 2655 | size_t size_read = 0; |
| 2656 | std::vector<uint8_t> temp_buffer; |
| 2657 | }; |
| 2658 | |
| 2659 | class llama_io_write_device : public llama_io_write_i { |
| 2660 | public: |
| 2661 | llama_io_write_device(uint8_t * p, size_t len, llama_memory_buffers & mbufs) : ptr(p), buf_size(len), mbufs(mbufs) { |
| 2662 | } |
| 2663 | |
| 2664 | ~llama_io_write_device() { |
| 2665 | llama_memory_buffers mbufs_new; |
| 2666 | |
| 2667 | for (const auto & winfo : winfos) { |
| 2668 | auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer); |
| 2669 | |
| 2670 | mbufs_new[buft].n_tensors++; |
| 2671 | mbufs_new[buft].total_size += winfo.size; |
| 2672 | } |
| 2673 | |
| 2674 | for (auto & [buft, mbuf] : mbufs_new) { |
| 2675 | ggml_init_params params = { |
| 2676 | /*.mem_size =*/ 2*mbuf.n_tensors*ggml_tensor_overhead(), |
| 2677 | /*.mem_buffer =*/ NULL__null, |
| 2678 | /*.no_alloc =*/ true, |
| 2679 | }; |
| 2680 | |
| 2681 | mbuf.ctx.reset(ggml_init(params)); |
| 2682 | |
| 2683 | mbuf.org.reserve(mbuf.n_tensors); |
| 2684 | mbuf.cpy.reserve(mbuf.n_tensors); |
| 2685 | } |
| 2686 | |
| 2687 | for (const auto & winfo : winfos) { |
| 2688 | auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer); |
| 2689 | |
| 2690 | const int64_t n = winfo.size/ggml_element_size(winfo.tensor); |
| 2691 | |
| 2692 | auto & mbuf = mbufs_new[buft]; |
| 2693 | |
| 2694 | mbuf.org.push_back(ggml_view_1d (mbuf.ctx.get(), winfo.tensor, n, winfo.offset)); |
| 2695 | mbuf.cpy.push_back(ggml_new_tensor_1d(mbuf.ctx.get(), winfo.tensor->type, n)); |
| 2696 | } |
| 2697 | |
| 2698 | for (auto & [buft, mbuf] : mbufs_new) { |
| 2699 | auto & mbuf_cur = mbufs[buft]; |
| 2700 | |
| 2701 | bool need_alloc = false; |
| 2702 | |
| 2703 | need_alloc = need_alloc || (!mbuf_cur.buf); |
| 2704 | need_alloc = need_alloc || (mbuf_cur.org.size() != mbuf.org.size()); |
| 2705 | need_alloc = need_alloc || (mbuf_cur.total_size != mbuf.total_size); |
| 2706 | |
| 2707 | if (!need_alloc) { |
| 2708 | for (size_t i = 0; i < mbuf_cur.org.size(); ++i) { |
| 2709 | auto * org0 = mbuf_cur.org[i]; |
| 2710 | auto * org1 = mbuf.org[i]; |
| 2711 | |
| 2712 | if (!ggml_are_same_shape(org0, org1)) { |
| 2713 | need_alloc = true; |
| 2714 | break; |
| 2715 | } |
| 2716 | |
| 2717 | if (org0->view_src != org1->view_src || org0->view_offs != org1->view_offs) { |
| 2718 | need_alloc = true; |
| 2719 | break; |
| 2720 | } |
| 2721 | } |
| 2722 | } |
| 2723 | |
| 2724 | if (need_alloc) { |
| 2725 | if (!mbuf_cur.buf || mbuf_cur.total_size != mbuf.total_size) { |
| 2726 | mbuf_cur = std::move(mbuf); |
| 2727 | |
| 2728 | mbuf_cur.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(mbuf_cur.ctx.get(), buft)); |
| 2729 | |
| 2730 | LLAMA_LOG_INFO("%s: allocated '%s' buffer %.3f MiB\n", __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0/1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: allocated '%s' buffer %.3f MiB\n" , __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0 /1024.0); |
| 2731 | } else { |
| 2732 | //LLAMA_LOG_INFO("%s: reallocating tensors in '%s' buffer %.3f MiB\n", __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0/1024.0); |
| 2733 | |
| 2734 | // save the old buffer and allocate the new tensors in it |
| 2735 | auto buf = std::move(mbuf_cur.buf); |
| 2736 | |
| 2737 | mbuf_cur = std::move(mbuf); |
| 2738 | |
| 2739 | ggml_tallocr talloc = ggml_tallocr_new(buf.get()); |
| 2740 | |
| 2741 | for (size_t i = 0; i < mbuf_cur.org.size(); ++i) { |
| 2742 | ggml_backend_view_init(mbuf_cur.org[i]); |
| 2743 | ggml_tallocr_alloc(&talloc, mbuf_cur.cpy[i]); |
| 2744 | } |
| 2745 | |
| 2746 | mbuf_cur.buf = std::move(buf); |
| 2747 | } |
| 2748 | } |
| 2749 | |
| 2750 | for (size_t i = 0; i < mbuf_cur.org.size(); ++i) { |
| 2751 | ggml_backend_tensor_copy(mbuf_cur.org[i], mbuf_cur.cpy[i]); |
| 2752 | } |
| 2753 | } |
| 2754 | } |
| 2755 | |
| 2756 | void write(const void * src, size_t size) override { |
| 2757 | if (size > buf_size) { |
| 2758 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2759 | } |
| 2760 | memcpy(ptr, src, size); |
| 2761 | ptr += size; |
| 2762 | size_written += size; |
| 2763 | buf_size -= size; |
| 2764 | } |
| 2765 | |
| 2766 | void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2767 | // save the write for later during destruction |
| 2768 | winfos.push_back({tensor, ptr, size, offset}); |
| 2769 | } |
| 2770 | |
| 2771 | size_t n_bytes() override { |
| 2772 | return size_written; |
| 2773 | } |
| 2774 | |
| 2775 | private: |
| 2776 | uint8_t * ptr; |
| 2777 | size_t buf_size = 0; |
| 2778 | size_t size_written = 0; |
| 2779 | |
| 2780 | struct write_info { |
| 2781 | ggml_tensor * tensor; |
| 2782 | uint8_t * ptr; |
| 2783 | size_t size; |
| 2784 | size_t offset; |
| 2785 | }; |
| 2786 | std::vector<write_info> winfos; |
| 2787 | |
| 2788 | llama_memory_buffers & mbufs; |
| 2789 | }; |
| 2790 | |
| 2791 | class llama_io_read_device : public llama_io_read_i { |
| 2792 | public: |
| 2793 | llama_io_read_device(const uint8_t * p, size_t len, const llama_memory_buffers & mbufs) : ptr(p), buf_size(len), mbufs(mbufs) { |
| 2794 | } |
| 2795 | |
| 2796 | ~llama_io_read_device() { |
| 2797 | llama_memory_buffers mbufs_new; |
| 2798 | |
| 2799 | for (const auto & rinfo : rinfos) { |
| 2800 | auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer); |
| 2801 | |
| 2802 | mbufs_new[buft].n_tensors++; |
| 2803 | mbufs_new[buft].total_size += rinfo.size; |
| 2804 | } |
| 2805 | |
| 2806 | for (auto & [buft, mbuf] : mbufs_new) { |
| 2807 | ggml_init_params params = { |
| 2808 | /*.mem_size =*/ mbuf.n_tensors*ggml_tensor_overhead(), |
| 2809 | /*.mem_buffer =*/ NULL__null, |
| 2810 | /*.no_alloc =*/ true, |
| 2811 | }; |
| 2812 | |
| 2813 | mbuf.ctx.reset(ggml_init(params)); |
| 2814 | |
| 2815 | mbuf.org.reserve(mbuf.n_tensors); |
| 2816 | } |
| 2817 | |
| 2818 | for (const auto & rinfo : rinfos) { |
| 2819 | auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer); |
| 2820 | |
| 2821 | const int64_t n = rinfo.size/ggml_element_size(rinfo.tensor); |
| 2822 | |
| 2823 | auto & mbuf = mbufs_new[buft]; |
| 2824 | |
| 2825 | mbuf.org.push_back(ggml_view_1d(mbuf.ctx.get(), rinfo.tensor, n, rinfo.offset)); |
| 2826 | |
| 2827 | ggml_backend_view_init(mbuf.org.back()); |
| 2828 | } |
| 2829 | |
| 2830 | for (auto & [buft, mbuf] : mbufs_new) { |
| 2831 | const auto & mbuf_cur = mbufs.at(buft); |
| 2832 | |
| 2833 | if (!mbuf_cur.buf || mbuf_cur.n_tensors != mbuf.n_tensors || mbuf_cur.total_size != mbuf.total_size) { |
| 2834 | GGML_ABORT("%s: memory buffer mismatch\n", __func__)ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2834, "%s: memory buffer mismatch\n", __func__); |
| 2835 | } |
| 2836 | |
| 2837 | for (size_t i = 0; i < mbuf_cur.org.size(); ++i) { |
| 2838 | ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]); |
| 2839 | } |
| 2840 | } |
| 2841 | |
| 2842 | GGML_ASSERT(buf_size == 0)if (!(buf_size == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2842, "GGML_ASSERT(%s) failed", "buf_size == 0"); |
| 2843 | } |
| 2844 | |
| 2845 | void read(void * dst, size_t size) override { |
| 2846 | if (size > buf_size) { |
| 2847 | throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer"); |
| 2848 | } |
| 2849 | memcpy(dst, ptr, size); |
| 2850 | ptr += size; |
| 2851 | size_read += size; |
| 2852 | buf_size -= size; |
| 2853 | } |
| 2854 | |
| 2855 | void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override { |
| 2856 | // save for later during destruction |
| 2857 | rinfos.push_back({tensor, ptr, size, offset}); |
| 2858 | } |
| 2859 | |
| 2860 | size_t n_bytes() override { |
| 2861 | return size_read; |
| 2862 | } |
| 2863 | |
| 2864 | private: |
| 2865 | const uint8_t * ptr; |
| 2866 | size_t buf_size = 0; |
| 2867 | size_t size_read = 0; |
| 2868 | |
| 2869 | struct read_info { |
| 2870 | ggml_tensor * tensor; |
| 2871 | const uint8_t * ptr; |
| 2872 | size_t size; |
| 2873 | size_t offset; |
| 2874 | }; |
| 2875 | std::vector<read_info> rinfos; |
| 2876 | |
| 2877 | const llama_memory_buffers & mbufs; |
| 2878 | }; |
| 2879 | |
| 2880 | size_t llama_context::state_get_size() { |
| 2881 | llama_io_write_dummy io(false); |
| 2882 | tryif (true) { |
| 2883 | return state_write_data(io); |
| 2884 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2885 | LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error getting state size: %s\n" , __func__, err.what()); |
| 2886 | return 0; |
| 2887 | } |
| 2888 | } |
| 2889 | |
| 2890 | size_t llama_context::state_get_data(uint8_t * dst, size_t size) { |
| 2891 | llama_io_write_host io(dst, size); |
| 2892 | tryif (true) { |
| 2893 | return state_write_data(io); |
| 2894 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2895 | LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving state: %s\n" , __func__, err.what()); |
| 2896 | return 0; |
| 2897 | } |
| 2898 | } |
| 2899 | |
| 2900 | size_t llama_context::state_set_data(const uint8_t * src, size_t size) { |
| 2901 | llama_io_read_host io(src, size); |
| 2902 | tryif (true) { |
| 2903 | return state_read_data(io); |
| 2904 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2905 | LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading state: %s\n" , __func__, err.what()); |
| 2906 | return 0; |
| 2907 | } |
| 2908 | } |
| 2909 | |
| 2910 | static constexpr uint32_t io_magic = 0xaf143cd8; |
| 2911 | |
| 2912 | size_t llama_context::state_seq_get_size(llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 2913 | llama_io_write_dummy io(flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2); |
| 2914 | tryif (true) { |
| 2915 | io.write(&io_magic, sizeof(io_magic)); |
| 2916 | io.write(&seq_id, sizeof(seq_id)); |
| 2917 | |
| 2918 | return state_seq_write_data(io, seq_id, flags); |
| 2919 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2920 | LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error getting state size: %s\n" , __func__, err.what()); |
| 2921 | return 0; |
| 2922 | } |
| 2923 | } |
| 2924 | |
| 2925 | size_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size, llama_state_seq_flags flags) { |
| 2926 | std::unique_ptr<llama_io_write_i> io; |
| 2927 | if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2) { |
| 2928 | io = std::make_unique<llama_io_write_device>(dst, size, mem_storage[seq_id]); |
| 2929 | } else { |
| 2930 | io = std::make_unique<llama_io_write_host>(dst, size); |
| 2931 | } |
| 2932 | |
| 2933 | tryif (true) { |
| 2934 | io->write(&io_magic, sizeof(io_magic)); |
| 2935 | io->write(&seq_id, sizeof(seq_id)); |
| 2936 | |
| 2937 | return state_seq_write_data(*io, seq_id, flags); |
| 2938 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2939 | LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving state: %s\n" , __func__, err.what()); |
| 2940 | return 0; |
| 2941 | } |
| 2942 | } |
| 2943 | |
| 2944 | size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size, llama_state_seq_flags flags) { |
| 2945 | std::unique_ptr<llama_io_read_i> io; |
| 2946 | if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2) { |
| 2947 | // create a temporary io to read the magic and the src seq_id |
| 2948 | io = std::make_unique<llama_io_read_host>(src, size); |
| 2949 | |
| 2950 | uint32_t magic_read; |
| 2951 | io->read(&magic_read, sizeof(magic_read)); |
| 2952 | if (io_magic != magic_read) { |
| 2953 | throwabort_with_suppression(); if (false) std::runtime_error("wrong sequence state magic"); |
| 2954 | } |
| 2955 | |
| 2956 | llama_seq_id seq_id_read; |
| 2957 | io->read(&seq_id_read, sizeof(seq_id_read)); |
| 2958 | |
| 2959 | GGML_ASSERT(mem_storage.find(seq_id_read) != mem_storage.end())if (!(mem_storage.find(seq_id_read) != mem_storage.end())) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2959, "GGML_ASSERT(%s) failed", "mem_storage.find(seq_id_read) != mem_storage.end()" ); |
| 2960 | |
| 2961 | io = std::make_unique<llama_io_read_device>(src, size, mem_storage[seq_id_read]); |
| 2962 | } else { |
| 2963 | io = std::make_unique<llama_io_read_host>(src, size); |
| 2964 | } |
| 2965 | |
| 2966 | tryif (true) { |
| 2967 | uint32_t magic_read; |
| 2968 | io->read(&magic_read, sizeof(magic_read)); |
| 2969 | if (io_magic != magic_read) { |
| 2970 | throwabort_with_suppression(); if (false) std::runtime_error("wrong sequence state magic"); |
| 2971 | } |
| 2972 | |
| 2973 | llama_seq_id seq_id_read; |
| 2974 | io->read(&seq_id_read, sizeof(seq_id_read)); |
| 2975 | |
| 2976 | return state_seq_read_data(*io, seq_id, flags); |
| 2977 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 2978 | LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading state: %s\n" , __func__, err.what()); |
| 2979 | return 0; |
| 2980 | } |
| 2981 | } |
| 2982 | |
| 2983 | bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { |
| 2984 | llama_file file(filepath, "rb"); |
| 2985 | |
| 2986 | // sanity checks |
| 2987 | { |
| 2988 | const uint32_t magic = file.read_u32(); |
| 2989 | const uint32_t version = file.read_u32(); |
| 2990 | |
| 2991 | if (magic != LLAMA_SESSION_MAGIC0x6767736eu || version != LLAMA_SESSION_VERSION9) { |
| 2992 | LLAMA_LOG_ERROR("%s: unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unknown (magic, version) for session file: %08x, %08x\n" , __func__, magic, version); |
| 2993 | return false; |
| 2994 | } |
| 2995 | } |
| 2996 | |
| 2997 | // load the prompt |
| 2998 | { |
| 2999 | const uint32_t n_token_count = file.read_u32(); |
| 3000 | |
| 3001 | if (n_token_count > n_token_capacity) { |
| 3002 | LLAMA_LOG_ERROR("%s: token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: token count in session file exceeded capacity! %u > %zu\n" , __func__, n_token_count, n_token_capacity); |
| 3003 | return false; |
| 3004 | } |
| 3005 | |
| 3006 | file.read_raw(tokens_out, sizeof(llama_token) * n_token_count); |
| 3007 | *n_token_count_out = n_token_count; |
| 3008 | } |
| 3009 | |
| 3010 | // restore the context state |
| 3011 | { |
| 3012 | const size_t n_state_size_cur = file.size() - file.tell(); |
| 3013 | |
| 3014 | llama_io_read_file io( &file); |
| 3015 | const size_t n_read = state_read_data(io); |
| 3016 | |
| 3017 | if (n_read != n_state_size_cur) { |
| 3018 | LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: did not read all of the session file data! size %zu, got %zu\n" , __func__, n_state_size_cur, n_read); |
| 3019 | return false; |
| 3020 | } |
| 3021 | } |
| 3022 | |
| 3023 | return true; |
| 3024 | } |
| 3025 | |
| 3026 | bool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) { |
| 3027 | llama_file file(filepath, "wb"); |
| 3028 | |
| 3029 | file.write_u32(LLAMA_SESSION_MAGIC0x6767736eu); |
| 3030 | file.write_u32(LLAMA_SESSION_VERSION9); |
| 3031 | |
| 3032 | // save the prompt |
| 3033 | file.write_u32((uint32_t) n_token_count); |
| 3034 | file.write_raw(tokens, sizeof(llama_token) * n_token_count); |
| 3035 | |
| 3036 | // save the context state using stream saving |
| 3037 | llama_io_write_file io(&file); |
| 3038 | state_write_data(io); |
| 3039 | |
| 3040 | return true; |
| 3041 | } |
| 3042 | |
| 3043 | size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { |
| 3044 | llama_file file(filepath, "rb"); |
| 3045 | |
| 3046 | // version checks |
| 3047 | { |
| 3048 | const uint32_t magic = file.read_u32(); |
| 3049 | const uint32_t version = file.read_u32(); |
| 3050 | |
| 3051 | if (magic != LLAMA_STATE_SEQ_MAGIC0x67677371u || version != LLAMA_STATE_SEQ_VERSION2) { |
| 3052 | LLAMA_LOG_ERROR("%s: unknown (magic, version) for sequence state file: %08x, %08x\n", __func__, magic, version)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unknown (magic, version) for sequence state file: %08x, %08x\n" , __func__, magic, version); |
| 3053 | return 0; |
| 3054 | } |
| 3055 | } |
| 3056 | |
| 3057 | // load the prompt |
| 3058 | { |
| 3059 | const uint32_t n_token_count = file.read_u32(); |
| 3060 | |
| 3061 | if (n_token_count > n_token_capacity) { |
| 3062 | LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: token count in sequence state file exceeded capacity! %u > %zu\n" , __func__, n_token_count, n_token_capacity); |
| 3063 | return 0; |
| 3064 | } |
| 3065 | |
| 3066 | file.read_raw(tokens_out, sizeof(llama_token) * n_token_count); |
| 3067 | *n_token_count_out = n_token_count; |
| 3068 | } |
| 3069 | |
| 3070 | // restore the context state |
| 3071 | { |
| 3072 | const size_t state_size = file.size() - file.tell(); |
| 3073 | llama_io_read_file io(&file); |
| 3074 | const size_t nread = state_seq_read_data(io, seq_id, 0); |
| 3075 | if (!nread) { |
| 3076 | LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to restore sequence state\n" , __func__); |
| 3077 | return 0; |
| 3078 | } |
| 3079 | GGML_ASSERT(nread <= state_size)if (!(nread <= state_size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3079, "GGML_ASSERT(%s) failed", "nread <= state_size"); |
| 3080 | GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell())if (!(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3080, "GGML_ASSERT(%s) failed", "nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * *n_token_count_out == file.tell()" ); |
| 3081 | } |
| 3082 | |
| 3083 | return file.tell(); |
| 3084 | } |
| 3085 | |
| 3086 | size_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) { |
| 3087 | llama_file file(filepath, "wb"); |
| 3088 | |
| 3089 | file.write_u32(LLAMA_STATE_SEQ_MAGIC0x67677371u); |
| 3090 | file.write_u32(LLAMA_STATE_SEQ_VERSION2); |
| 3091 | |
| 3092 | // save the prompt |
| 3093 | file.write_u32((uint32_t) n_token_count); |
| 3094 | file.write_raw(tokens, sizeof(llama_token) * n_token_count); |
| 3095 | |
| 3096 | // save the context state using stream saving |
| 3097 | llama_io_write_file io(&file); |
| 3098 | state_seq_write_data(io, seq_id, 0); |
| 3099 | |
| 3100 | const size_t res = file.tell(); |
| 3101 | GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes())if (!(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3101, "GGML_ASSERT(%s) failed", "res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes()" ); |
| 3102 | |
| 3103 | return res; |
| 3104 | } |
| 3105 | |
| 3106 | size_t llama_context::state_write_data(llama_io_write_i & io) { |
| 3107 | LLAMA_LOG_DEBUG("%s: writing state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: writing state\n" , __func__); |
| 3108 | |
| 3109 | // write model info |
| 3110 | { |
| 3111 | LLAMA_LOG_DEBUG("%s: - writing model info\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - writing model info\n" , __func__); |
| 3112 | |
| 3113 | const std::string arch_str = llm_arch_name(model.arch); |
| 3114 | io.write_string(arch_str); |
| 3115 | // TODO: add more model-specific info which should prevent loading the session file if not identical |
| 3116 | } |
| 3117 | |
| 3118 | if (memory != nullptr) { |
| 3119 | LLAMA_LOG_DEBUG("%s: - writing memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - writing memory module\n" , __func__); |
| 3120 | memory->state_write(io); |
| 3121 | } |
| 3122 | |
| 3123 | return io.n_bytes(); |
| 3124 | } |
| 3125 | |
| 3126 | size_t llama_context::state_read_data(llama_io_read_i & io) { |
| 3127 | LLAMA_LOG_DEBUG("%s: reading state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reading state\n" , __func__); |
| 3128 | |
| 3129 | // read model info |
| 3130 | { |
| 3131 | LLAMA_LOG_DEBUG("%s: - reading model info\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - reading model info\n" , __func__); |
| 3132 | |
| 3133 | const std::string cur_arch_str = llm_arch_name(model.arch); |
| 3134 | |
| 3135 | std::string arch_str; |
| 3136 | io.read_string(arch_str); |
| 3137 | if (cur_arch_str != arch_str) { |
| 3138 | throwabort_with_suppression(); if (false) std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str())); |
| 3139 | } |
| 3140 | // TODO: add more info which needs to be identical but which is not verified otherwise |
| 3141 | } |
| 3142 | |
| 3143 | if (memory) { |
| 3144 | LLAMA_LOG_DEBUG("%s: - reading memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - reading memory module\n" , __func__); |
| 3145 | |
| 3146 | memory->state_read(io); |
| 3147 | } |
| 3148 | |
| 3149 | return io.n_bytes(); |
| 3150 | } |
| 3151 | |
| 3152 | size_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 3153 | GGML_UNUSED(seq_id)(void)(seq_id); |
| 3154 | |
| 3155 | if (memory) { |
| 3156 | memory->state_write(io, seq_id, flags); |
| 3157 | } |
| 3158 | |
| 3159 | return io.n_bytes(); |
| 3160 | } |
| 3161 | |
| 3162 | size_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 3163 | GGML_UNUSED(seq_id)(void)(seq_id); |
| 3164 | |
| 3165 | if (memory) { |
| 3166 | memory->state_read(io, seq_id, flags); |
| 3167 | } |
| 3168 | |
| 3169 | return io.n_bytes(); |
| 3170 | } |
| 3171 | |
| 3172 | // |
| 3173 | // perf |
| 3174 | // |
| 3175 | |
| 3176 | llama_perf_context_data llama_context::perf_get_data() const { |
| 3177 | llama_perf_context_data data = {}; |
| 3178 | |
| 3179 | data.t_start_ms = 1e-3 * t_start_us; |
| 3180 | data.t_load_ms = 1e-3 * t_load_us; |
| 3181 | data.t_p_eval_ms = 1e-3 * t_p_eval_us; |
| 3182 | data.t_eval_ms = 1e-3 * t_eval_us; |
| 3183 | data.n_p_eval = std::max(1, n_p_eval); |
| 3184 | data.n_eval = std::max(1, n_eval); |
| 3185 | data.n_reused = std::max(0, n_reused); |
| 3186 | |
| 3187 | return data; |
| 3188 | } |
| 3189 | |
| 3190 | void llama_context::perf_reset() { |
| 3191 | t_start_us = ggml_time_us(); |
| 3192 | t_eval_us = n_eval = 0; |
| 3193 | t_p_eval_us = n_p_eval = 0; |
| 3194 | n_reused = 0; |
| 3195 | } |
| 3196 | |
| 3197 | llama_memory_breakdown llama_context::memory_breakdown() const { |
| 3198 | std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> ret; |
| 3199 | for (const auto & [buft, size] : model.memory_breakdown()) { |
| 3200 | ret[buft].model += size; |
| 3201 | } |
| 3202 | if (memory) { |
| 3203 | for (const auto & [buft, size] : memory->memory_breakdown()) { |
| 3204 | ret[buft].context += size; |
| 3205 | } |
| 3206 | } |
| 3207 | if (model.hparams.no_alloc) { |
| 3208 | for (size_t i = 0; i < backends.size(); ++i) { |
| 3209 | ggml_backend_t backend = backends[i].get(); |
| 3210 | ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); |
| 3211 | ret[buft].compute += backend_buf_exp_size[i]; |
| 3212 | } |
| 3213 | } else { |
| 3214 | for (const auto & backend_ptr : backends) { |
| 3215 | ggml_backend_t backend = backend_ptr.get(); |
| 3216 | ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend); |
| 3217 | ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend); |
| 3218 | } |
| 3219 | } |
| 3220 | return ret; |
| 3221 | } |
| 3222 | |
| 3223 | // |
| 3224 | // training |
| 3225 | // |
| 3226 | |
| 3227 | static void llama_set_param(struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) { |
| 3228 | if (!tensor || tensor->type != GGML_TYPE_F32) { |
| 3229 | return; |
| 3230 | } |
| 3231 | if (!param_filter(tensor, userdata)) { |
| 3232 | return; |
| 3233 | } |
| 3234 | if (strcmp(tensor->name, "token_embd.weight") == 0) { |
| 3235 | return; // FIXME |
| 3236 | } |
| 3237 | if (strcmp(tensor->name, "rope_freqs.weight") == 0) { |
| 3238 | return; // FIXME |
| 3239 | } |
| 3240 | ggml_set_param(tensor); |
| 3241 | } |
| 3242 | |
| 3243 | void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) { |
| 3244 | GGML_ASSERT(!opt_ctx)if (!(!opt_ctx)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3244, "GGML_ASSERT(%s) failed", "!opt_ctx"); |
| 3245 | model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx(); |
| 3246 | const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train); |
| 3247 | const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch); |
| 3248 | GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0)if (!(model->hparams.n_ctx_train % n_batch == 0)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3248, "GGML_ASSERT(%s) failed", "model->hparams.n_ctx_train % n_batch == 0" ); |
| 3249 | GGML_ASSERT(n_batch % n_ubatch == 0)if (!(n_batch % n_ubatch == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3249, "GGML_ASSERT(%s) failed", "n_batch % n_ubatch == 0"); |
| 3250 | |
| 3251 | ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY); |
| 3252 | opt_params.opt_period = n_batch / n_ubatch; |
| 3253 | opt_params.get_opt_pars = lopt_params.get_opt_pars; |
| 3254 | opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud; |
| 3255 | opt_params.optimizer = lopt_params.optimizer_type; |
| 3256 | opt_ctx = ggml_opt_init(opt_params); |
| 3257 | |
| 3258 | llama_opt_param_filter param_filter = lopt_params.param_filter; |
| 3259 | void * param_filter_ud = lopt_params.param_filter_ud; |
| 3260 | |
| 3261 | //llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME |
| 3262 | llama_set_param(model->type_embd, param_filter, param_filter_ud); |
| 3263 | llama_set_param(model->pos_embd, param_filter, param_filter_ud); |
| 3264 | llama_set_param(model->tok_norm, param_filter, param_filter_ud); |
| 3265 | llama_set_param(model->tok_norm_b, param_filter, param_filter_ud); |
| 3266 | llama_set_param(model->output_norm, param_filter, param_filter_ud); |
| 3267 | llama_set_param(model->output_norm_b, param_filter, param_filter_ud); |
| 3268 | llama_set_param(model->output, param_filter, param_filter_ud); |
| 3269 | llama_set_param(model->output_b, param_filter, param_filter_ud); |
| 3270 | llama_set_param(model->output_norm_enc, param_filter, param_filter_ud); |
| 3271 | llama_set_param(model->cls, param_filter, param_filter_ud); |
| 3272 | llama_set_param(model->cls_b, param_filter, param_filter_ud); |
| 3273 | llama_set_param(model->cls_out, param_filter, param_filter_ud); |
| 3274 | llama_set_param(model->cls_out_b, param_filter, param_filter_ud); |
| 3275 | llama_set_param(model->cls_norm, param_filter, param_filter_ud); |
| 3276 | |
| 3277 | for (struct llama_layer & layer : model->layers) { |
| 3278 | for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) { |
| 3279 | llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud); |
| 3280 | } |
| 3281 | } |
| 3282 | } |
| 3283 | |
| 3284 | void llama_context::opt_epoch_iter( |
| 3285 | ggml_opt_dataset_t dataset, |
| 3286 | ggml_opt_result_t result, |
| 3287 | const std::vector<llama_token> & tokens, |
| 3288 | const std::vector<llama_token> & labels_sparse, |
| 3289 | llama_batch & batch, |
| 3290 | ggml_opt_epoch_callback callback, |
| 3291 | bool train, |
| 3292 | int64_t idata_in_loop, |
| 3293 | int64_t ndata_in_loop, |
| 3294 | int64_t t_loop_start) { |
| 3295 | GGML_ASSERT(opt_ctx)if (!(opt_ctx)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3295, "GGML_ASSERT(%s) failed", "opt_ctx"); |
| 3296 | const uint32_t n_ctx = llama_model_n_ctx_train(&model); |
| 3297 | const uint32_t n_batch = std::min(this->n_batch(), n_ctx); |
| 3298 | const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch); |
| 3299 | |
| 3300 | memory->clear(true); |
| 3301 | |
| 3302 | for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) { |
| 3303 | batch.n_tokens = n_batch; |
| 3304 | for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) { |
| 3305 | batch.token [pos_batch] = tokens[pos_ctx + pos_batch]; |
| 3306 | batch.pos [pos_batch] = pos_ctx + pos_batch; |
| 3307 | batch.n_seq_id[pos_batch] = 1; |
| 3308 | batch.seq_id [pos_batch][0] = 0; |
| 3309 | batch.logits [pos_batch] = true; |
| 3310 | } |
| 3311 | |
| 3312 | if (!balloc->init(batch, model.vocab, nullptr, model.hparams.n_embd_inp(), cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max, true)) { |
| 3313 | LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__); |
| 3314 | return; |
| 3315 | } |
| 3316 | |
| 3317 | const uint32_t n_tokens_all = balloc->get_n_tokens(); |
| 3318 | |
| 3319 | n_queued_tokens += n_tokens_all; |
| 3320 | |
| 3321 | embd_seq.clear(); |
| 3322 | |
| 3323 | uint32_t n_outputs_all = n_tokens_all; |
| 3324 | |
| 3325 | auto mctx = memory->init_batch(*balloc, cparams.n_ubatch, true); |
| 3326 | if (!mctx || mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) { |
| 3327 | LLAMA_LOG_ERROR("%s: could not initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not initialize batch\n" , __func__); |
| 3328 | break; |
| 3329 | } |
| 3330 | |
| 3331 | // reserve output buffer |
| 3332 | if (output_reserve(n_outputs_all) < n_outputs_all) { |
| 3333 | LLAMA_LOG_ERROR("%s: could not reserve space for batch with %d outputs\n", __func__, n_outputs_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %d outputs\n" , __func__, n_outputs_all); |
| 3334 | GGML_ABORT("TODO: handle this error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3334, "TODO: handle this error"); |
| 3335 | }; |
| 3336 | |
| 3337 | uint32_t pos_batch = 0; |
| 3338 | do { |
| 3339 | const auto & ubatch = mctx->get_ubatch(); |
| 3340 | |
| 3341 | n_outputs = ubatch.n_tokens; |
| 3342 | |
| 3343 | if (!mctx->apply()) { |
| 3344 | LLAMA_LOG_ERROR("%s: failed to update the memory context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to update the memory context\n" , __func__); |
| 3345 | break; |
| 3346 | } |
| 3347 | |
| 3348 | auto * res = gf_res_prev.get(); |
| 3349 | |
| 3350 | const auto gparams = graph_params(res, ubatch, mctx.get(), ctx_type_to_graph_type(cparams.ctx_type)); |
| 3351 | |
| 3352 | res->reset(); |
| 3353 | |
| 3354 | auto * gf = model.build_graph(gparams); |
| 3355 | |
| 3356 | struct ggml_context * ctx_compute_opt; |
| 3357 | { |
| 3358 | const size_t size_gf = ggml_graph_size(gf); |
| 3359 | const size_t size_meta = 4*size_gf*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(size_gf, /*grads = */ true); |
| 3360 | struct ggml_init_params params = { |
| 3361 | /*.mem_size =*/ size_meta, |
| 3362 | /*.mem_buffer =*/ nullptr, |
| 3363 | /*.no_alloc =*/ true, |
| 3364 | }; |
| 3365 | ctx_compute_opt = ggml_init(params); |
| 3366 | } |
| 3367 | ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_inp_tokens(), res->get_logits()); |
| 3368 | ggml_opt_alloc(opt_ctx, train); |
| 3369 | |
| 3370 | res->set_inputs(&ubatch); |
| 3371 | { |
| 3372 | struct ggml_tensor * labels = ggml_opt_labels(opt_ctx); |
| 3373 | GGML_ASSERT(labels->ne[1] == n_ubatch)if (!(labels->ne[1] == n_ubatch)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3373, "GGML_ASSERT(%s) failed", "labels->ne[1] == n_ubatch" ); |
| 3374 | ggml_set_zero(labels); |
| 3375 | const float onef = 1.0f; |
| 3376 | for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) { |
| 3377 | const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch; |
| 3378 | GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0])if (!(labels_sparse[ilabel] < labels->ne[0])) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3378, "GGML_ASSERT(%s) failed", "labels_sparse[ilabel] < labels->ne[0]" ); |
| 3379 | ggml_backend_tensor_set(labels, &onef, (pos_ubatch*labels->ne[0] + labels_sparse[ilabel])*sizeof(float), sizeof(float)); |
| 3380 | } |
| 3381 | } |
| 3382 | ggml_opt_eval(opt_ctx, result); |
| 3383 | if (callback) { |
| 3384 | callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start); |
| 3385 | } |
| 3386 | ggml_free(ctx_compute_opt); |
| 3387 | |
| 3388 | pos_batch += ubatch.n_tokens; |
| 3389 | } while (mctx->next()); |
| 3390 | } |
| 3391 | } |
| 3392 | |
| 3393 | void llama_context::opt_epoch( |
| 3394 | ggml_opt_dataset_t dataset, |
| 3395 | ggml_opt_result_t result_train, |
| 3396 | ggml_opt_result_t result_eval, |
| 3397 | int64_t idata_split, |
| 3398 | ggml_opt_epoch_callback callback_train, |
| 3399 | ggml_opt_epoch_callback callback_eval) { |
| 3400 | const uint32_t n_ctx = this->n_ctx(); |
| 3401 | const uint32_t n_batch = std::min(cparams.n_batch, n_ctx); |
| 3402 | const uint32_t n_ubatch = std::min(cparams.n_ubatch, n_batch); |
| 3403 | const int64_t ndata = ggml_opt_dataset_ndata(dataset); |
| 3404 | |
| 3405 | GGML_ASSERT(idata_split >= 0)if (!(idata_split >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3405, "GGML_ASSERT(%s) failed", "idata_split >= 0"); |
| 3406 | GGML_ASSERT(idata_split <= ndata)if (!(idata_split <= ndata)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3406, "GGML_ASSERT(%s) failed", "idata_split <= ndata"); |
| 3407 | |
| 3408 | const uint32_t ubatch_per_ctx = n_ctx / n_ubatch; |
| 3409 | |
| 3410 | struct llama_batch batch = llama_batch_init(n_batch, 0, 1); |
| 3411 | std::vector<llama_token> tokens(n_ctx); |
| 3412 | std::vector<llama_token> labels_sparse(n_ctx); |
| 3413 | |
| 3414 | int64_t idata = 0; |
| 3415 | |
| 3416 | int64_t t_loop_start = ggml_time_us(); |
| 3417 | int64_t ndata_in_loop = idata_split*ubatch_per_ctx; |
| 3418 | for (; idata < idata_split; ++idata) { |
| 3419 | constexpr bool train = true; |
| 3420 | const int64_t idata_in_loop = idata*ubatch_per_ctx; |
| 3421 | |
| 3422 | ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata); |
| 3423 | opt_epoch_iter(dataset, result_train, tokens, labels_sparse, batch, |
| 3424 | callback_train, train, idata_in_loop, ndata_in_loop, t_loop_start); |
| 3425 | } |
| 3426 | |
| 3427 | t_loop_start = ggml_time_us(); |
| 3428 | ndata_in_loop = (ndata - idata_split)*ubatch_per_ctx; |
| 3429 | for (; idata < ndata; ++idata) { |
| 3430 | constexpr bool train = false; |
| 3431 | const int64_t idata_in_loop = (idata - idata_split)*ubatch_per_ctx; |
| 3432 | |
| 3433 | ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata); |
| 3434 | opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch, |
| 3435 | callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start); |
| 3436 | } |
| 3437 | |
| 3438 | llama_batch_free(batch); |
| 3439 | } |
| 3440 | |
| 3441 | // |
| 3442 | // interface implementation |
| 3443 | // |
| 3444 | |
| 3445 | llama_context_params llama_context_default_params() { |
| 3446 | llama_context_params result = { |
| 3447 | /*.n_ctx =*/ 512, |
| 3448 | /*.n_batch =*/ 2048, |
| 3449 | /*.n_ubatch =*/ 512, |
| 3450 | /*.n_seq_max =*/ 1, |
| 3451 | /*.n_rs_seq =*/ 0, |
| 3452 | /*.n_outputs_max =*/ 0, |
| 3453 | /*.n_threads =*/ GGML_DEFAULT_N_THREADS4, // TODO: better default |
| 3454 | /*.n_threads_batch =*/ GGML_DEFAULT_N_THREADS4, |
| 3455 | /*.ctx_type =*/ LLAMA_CONTEXT_TYPE_DEFAULT, |
| 3456 | /*.rope_scaling_type =*/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, |
| 3457 | /*.pooling_type =*/ LLAMA_POOLING_TYPE_UNSPECIFIED, |
| 3458 | /*.attention_type =*/ LLAMA_ATTENTION_TYPE_UNSPECIFIED, |
| 3459 | /*.flash_attn_type =*/ LLAMA_FLASH_ATTN_TYPE_AUTO, |
| 3460 | /*.rope_freq_base =*/ 0.0f, |
| 3461 | /*.rope_freq_scale =*/ 0.0f, |
| 3462 | /*.yarn_ext_factor =*/ -1.0f, |
| 3463 | /*.yarn_attn_factor =*/ -1.0f, |
| 3464 | /*.yarn_beta_fast =*/ -1.0f, |
| 3465 | /*.yarn_beta_slow =*/ -1.0f, |
| 3466 | /*.yarn_orig_ctx =*/ 0, |
| 3467 | /*.defrag_thold =*/ -1.0f, |
| 3468 | /*.cb_eval =*/ nullptr, |
| 3469 | /*.cb_eval_user_data =*/ nullptr, |
| 3470 | /*.type_k =*/ GGML_TYPE_F16, |
| 3471 | /*.type_v =*/ GGML_TYPE_F16, |
| 3472 | /*.abort_callback =*/ nullptr, |
| 3473 | /*.abort_callback_data =*/ nullptr, |
| 3474 | /*.embeddings =*/ false, |
| 3475 | /*.offload_kqv =*/ true, |
| 3476 | /*.no_perf =*/ true, |
| 3477 | /*.op_offload =*/ true, |
| 3478 | /*.swa_full =*/ true, |
| 3479 | /*.kv_unified =*/ false, |
| 3480 | /*.sampler =*/ nullptr, |
| 3481 | /*.n_sampler =*/ 0, |
| 3482 | /*.ctx_other =*/ nullptr, |
| 3483 | }; |
| 3484 | |
| 3485 | return result; |
| 3486 | } |
| 3487 | |
| 3488 | llama_context * llama_init_from_model( |
| 3489 | llama_model * model, |
| 3490 | llama_context_params params) { |
| 3491 | if (!model) { |
| 3492 | LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: model cannot be NULL\n" , __func__); |
| 3493 | return nullptr; |
| 3494 | } |
| 3495 | |
| 3496 | if (params.n_batch == 0 && params.n_ubatch == 0) { |
| 3497 | LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_batch and n_ubatch cannot both be zero\n" , __func__); |
| 3498 | return nullptr; |
| 3499 | } |
| 3500 | |
| 3501 | if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) { |
| 3502 | LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n" , __func__); |
| 3503 | return nullptr; |
| 3504 | } |
| 3505 | |
| 3506 | if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && model->arch == LLM_ARCH_GROK) { |
| 3507 | LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: flash_attn is not compatible with Grok - forcing off\n" , __func__); |
| 3508 | params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED; |
| 3509 | } |
| 3510 | |
| 3511 | if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) { |
| 3512 | if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) { |
| 3513 | LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for SPLIT_MODE_TENSOR\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: enabling flash_attn since it is required for SPLIT_MODE_TENSOR\n" , __func__); |
| 3514 | params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; |
| 3515 | } |
| 3516 | if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) { |
| 3517 | LLAMA_LOG_ERROR("%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n" , __func__); |
| 3518 | return nullptr; |
| 3519 | } |
| 3520 | } |
| 3521 | |
| 3522 | if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) { |
| 3523 | const uint32_t blck_size = ggml_blck_size(params.type_k); |
| 3524 | for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { |
| 3525 | if (model->hparams.n_embd_head_k(il) % blck_size != 0) { |
| 3526 | LLAMA_LOG_ERROR("%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n" , __func__, ggml_type_name(params.type_k), blck_size, model-> hparams.n_embd_head_k(il)) |
| 3527 | __func__, ggml_type_name(params.type_k), blck_size, model->hparams.n_embd_head_k(il))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n" , __func__, ggml_type_name(params.type_k), blck_size, model-> hparams.n_embd_head_k(il)); |
| 3528 | return nullptr; |
| 3529 | } |
| 3530 | } |
| 3531 | } |
| 3532 | |
| 3533 | if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_v)) { |
| 3534 | const uint32_t blck_size = ggml_blck_size(params.type_v); |
| 3535 | for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { |
| 3536 | if (model->hparams.n_embd_head_v(il) % blck_size != 0) { |
| 3537 | LLAMA_LOG_ERROR("%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n" , __func__, ggml_type_name(params.type_v), blck_size, model-> hparams.n_embd_head_v(il)) |
| 3538 | __func__, ggml_type_name(params.type_v), blck_size, model->hparams.n_embd_head_v(il))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n" , __func__, ggml_type_name(params.type_v), blck_size, model-> hparams.n_embd_head_v(il)); |
| 3539 | return nullptr; |
| 3540 | } |
| 3541 | } |
| 3542 | } |
| 3543 | |
| 3544 | if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) { |
| 3545 | LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache quantization requires flash_attn\n" , __func__); |
| 3546 | return nullptr; |
| 3547 | } |
| 3548 | |
| 3549 | if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && |
| 3550 | params.pooling_type != model->hparams.pooling_type) { |
| 3551 | //user-specified pooling-type is different from the model default |
| 3552 | LLAMA_LOG_WARN("%s: model default pooling_type is [%d], but [%d] was specified\n", __func__,llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: model default pooling_type is [%d], but [%d] was specified\n" , __func__, model->hparams.pooling_type, params.pooling_type ) |
| 3553 | model->hparams.pooling_type, params.pooling_type)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: model default pooling_type is [%d], but [%d] was specified\n" , __func__, model->hparams.pooling_type, params.pooling_type ); |
| 3554 | } |
| 3555 | |
| 3556 | if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && |
| 3557 | model->hparams.n_layer_nextn == 0) { |
| 3558 | LLAMA_LOG_WARN("%s: context type MTP requested but model doesn't contain MTP layers\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: context type MTP requested but model doesn't contain MTP layers\n" , __func__); |
| 3559 | return nullptr; |
| 3560 | } |
| 3561 | |
| 3562 | tryif (true) { |
| 3563 | auto * ctx = new llama_context(*model, params); |
| 3564 | return ctx; |
| 3565 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 3566 | LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize the context: %s\n" , __func__, err.what()); |
| 3567 | } |
| 3568 | |
| 3569 | return nullptr; |
| 3570 | } |
| 3571 | |
| 3572 | // deprecated |
| 3573 | llama_context * llama_new_context_with_model( |
| 3574 | llama_model * model, |
| 3575 | llama_context_params params) { |
| 3576 | return llama_init_from_model(model, params); |
| 3577 | } |
| 3578 | |
| 3579 | void llama_free(llama_context * ctx) { |
| 3580 | delete ctx; |
| 3581 | } |
| 3582 | |
| 3583 | uint32_t llama_n_ctx(const llama_context * ctx) { |
| 3584 | return ctx->n_ctx(); |
| 3585 | } |
| 3586 | |
| 3587 | uint32_t llama_n_ctx_seq(const llama_context * ctx) { |
| 3588 | return ctx->n_ctx_seq(); |
| 3589 | } |
| 3590 | |
| 3591 | uint32_t llama_n_batch(const llama_context * ctx) { |
| 3592 | return ctx->n_batch(); |
| 3593 | } |
| 3594 | |
| 3595 | uint32_t llama_n_ubatch(const llama_context * ctx) { |
| 3596 | return ctx->n_ubatch(); |
| 3597 | } |
| 3598 | |
| 3599 | uint32_t llama_n_seq_max(const llama_context * ctx) { |
| 3600 | return ctx->n_seq_max(); |
| 3601 | } |
| 3602 | |
| 3603 | uint32_t llama_n_rs_seq(const llama_context * ctx) { |
| 3604 | return ctx->get_cparams().n_rs_seq; |
| 3605 | } |
| 3606 | |
| 3607 | const llama_model * llama_get_model(const llama_context * ctx) { |
| 3608 | return &ctx->get_model(); |
| 3609 | } |
| 3610 | |
| 3611 | enum llama_pooling_type llama_pooling_type(const llama_context * ctx) { |
| 3612 | return ctx->pooling_type(); |
| 3613 | } |
| 3614 | |
| 3615 | void llama_attach_threadpool( |
| 3616 | llama_context * ctx, |
| 3617 | ggml_threadpool_t threadpool, |
| 3618 | ggml_threadpool_t threadpool_batch) { |
| 3619 | ctx->attach_threadpool(threadpool, threadpool_batch); |
| 3620 | } |
| 3621 | |
| 3622 | void llama_detach_threadpool(llama_context * ctx) { |
| 3623 | ctx->detach_threadpool(); |
| 3624 | } |
| 3625 | |
| 3626 | void llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) { |
| 3627 | ctx->set_n_threads(n_threads, n_threads_batch); |
| 3628 | } |
| 3629 | |
| 3630 | int32_t llama_n_threads(llama_context * ctx) { |
| 3631 | return ctx->n_threads(); |
| 3632 | } |
| 3633 | |
| 3634 | int32_t llama_n_threads_batch(llama_context * ctx) { |
| 3635 | return ctx->n_threads_batch(); |
| 3636 | } |
| 3637 | |
| 3638 | void llama_set_abort_callback(llama_context * ctx, bool (*abort_callback)(void * data), void * abort_callback_data) { |
| 3639 | ctx->set_abort_callback(abort_callback, abort_callback_data); |
| 3640 | } |
| 3641 | |
| 3642 | void llama_set_embeddings(llama_context * ctx, bool embeddings) { |
| 3643 | ctx->set_embeddings(embeddings); |
| 3644 | } |
| 3645 | |
| 3646 | void llama_set_causal_attn(llama_context * ctx, bool causal_attn) { |
| 3647 | ctx->set_causal_attn(causal_attn); |
| 3648 | } |
| 3649 | |
| 3650 | void llama_set_warmup(llama_context * ctx, bool warmup) { |
| 3651 | ctx->set_warmup(warmup); |
| 3652 | } |
| 3653 | |
| 3654 | void llama_synchronize(llama_context * ctx) { |
| 3655 | ctx->synchronize(); |
| 3656 | } |
| 3657 | |
| 3658 | float * llama_get_logits(llama_context * ctx) { |
| 3659 | ctx->synchronize(); |
| 3660 | |
| 3661 | return ctx->get_logits(); |
| 3662 | } |
| 3663 | |
| 3664 | float * llama_get_logits_ith(llama_context * ctx, int32_t i) { |
| 3665 | ctx->synchronize(); |
| 3666 | |
| 3667 | float * res = nullptr; |
| 3668 | |
| 3669 | res = ctx->get_sampled_logits_ith(i); |
| 3670 | |
| 3671 | if (!res) { |
| 3672 | res = ctx->get_logits_ith(i); |
| 3673 | } |
| 3674 | |
| 3675 | return res; |
| 3676 | } |
| 3677 | |
| 3678 | float * llama_get_embeddings(llama_context * ctx) { |
| 3679 | ctx->synchronize(); |
| 3680 | |
| 3681 | return ctx->get_embeddings(); |
| 3682 | } |
| 3683 | |
| 3684 | float * llama_get_embeddings_ith(llama_context * ctx, int32_t i) { |
| 3685 | ctx->synchronize(); |
| 3686 | |
| 3687 | return ctx->get_embeddings_ith(i); |
| 3688 | } |
| 3689 | |
| 3690 | float * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) { |
| 3691 | ctx->synchronize(); |
| 3692 | |
| 3693 | return ctx->get_embeddings_seq(seq_id); |
| 3694 | } |
| 3695 | |
| 3696 | void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) { |
| 3697 | ctx->set_embeddings_nextn(value, masked); |
| 3698 | } |
| 3699 | |
| 3700 | void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) { |
| 3701 | ctx->set_embeddings_layer_inp(lid, value); |
| 3702 | } |
| 3703 | |
| 3704 | llama_memory_t llama_get_memory(const struct llama_context * ctx) { |
| 3705 | if (!ctx) { |
| 3706 | return nullptr; |
| 3707 | } |
| 3708 | |
| 3709 | return ctx->get_memory(); |
| 3710 | } |
| 3711 | |
| 3712 | float * llama_get_embeddings_nextn(llama_context * ctx) { |
| 3713 | ctx->synchronize(); |
| 3714 | |
| 3715 | return ctx->get_embeddings_nextn(); |
| 3716 | } |
| 3717 | |
| 3718 | float * llama_get_embeddings_nextn_ith(llama_context * ctx, int32_t i) { |
| 3719 | ctx->synchronize(); |
| 3720 | |
| 3721 | return ctx->get_embeddings_nextn_ith(i); |
| 3722 | } |
| 3723 | |
| 3724 | float * llama_get_embeddings_layer_inp(llama_context * ctx, uint32_t lid) { |
| 3725 | ctx->synchronize(); |
| 3726 | |
| 3727 | return ctx->get_embeddings_layer_inp(lid); |
| 3728 | } |
| 3729 | |
| 3730 | bool llama_set_sampler(llama_context * ctx, llama_seq_id seq_id, llama_sampler * smpl) { |
| 3731 | return ctx->set_sampler(seq_id, smpl); |
| 3732 | } |
| 3733 | |
| 3734 | llama_token llama_get_sampled_token_ith(llama_context * ctx, int32_t i) { |
| 3735 | ctx->synchronize(); |
| 3736 | |
| 3737 | return ctx->get_sampled_token_ith(i); |
| 3738 | } |
| 3739 | |
| 3740 | float * llama_get_sampled_probs_ith(llama_context * ctx, int32_t i) { |
| 3741 | ctx->synchronize(); |
| 3742 | |
| 3743 | return ctx->get_sampled_probs_ith(i); |
| 3744 | } |
| 3745 | |
| 3746 | float * llama_get_sampled_logits_ith(llama_context * ctx, int32_t i) { |
| 3747 | ctx->synchronize(); |
| 3748 | |
| 3749 | return ctx->get_sampled_logits_ith(i); |
| 3750 | } |
| 3751 | |
| 3752 | llama_token * llama_get_sampled_candidates_ith(llama_context * ctx, int32_t i) { |
| 3753 | ctx->synchronize(); |
| 3754 | |
| 3755 | return const_cast<llama_token *>(ctx->get_sampled_candidates_ith(i)); |
| 3756 | } |
| 3757 | |
| 3758 | uint32_t llama_get_sampled_candidates_count_ith(llama_context * ctx, int32_t i) { |
| 3759 | ctx->synchronize(); |
| 3760 | |
| 3761 | return static_cast<uint32_t>(ctx->get_sampled_candidates_count(i)); |
| 3762 | } |
| 3763 | |
| 3764 | uint32_t llama_get_sampled_logits_count_ith(llama_context * ctx, int32_t i) { |
| 3765 | ctx->synchronize(); |
| 3766 | |
| 3767 | return static_cast<uint32_t>(ctx->get_sampled_logits_count(i)); |
| 3768 | } |
| 3769 | |
| 3770 | uint32_t llama_get_sampled_probs_count_ith(llama_context * ctx, int32_t i) { |
| 3771 | ctx->synchronize(); |
| 3772 | |
| 3773 | return static_cast<uint32_t>(ctx->get_sampled_probs_count(i)); |
| 3774 | } |
| 3775 | |
| 3776 | struct ggml_cgraph * llama_graph_reserve( |
| 3777 | struct llama_context * ctx, |
| 3778 | uint32_t n_tokens, |
| 3779 | uint32_t n_seqs, |
| 3780 | uint32_t n_outputs) { |
| 3781 | auto memory = ctx->get_memory(); |
| 3782 | llama_memory_context_ptr mctx; |
| 3783 | if (memory) { |
| 3784 | mctx = memory->init_full(); |
| 3785 | } |
| 3786 | return ctx->graph_reserve(n_tokens, n_seqs, n_outputs, mctx.get()); |
| 3787 | } |
| 3788 | |
| 3789 | // llama adapter API |
| 3790 | |
| 3791 | int32_t llama_set_adapters_lora( |
| 3792 | llama_context * ctx, |
| 3793 | llama_adapter_lora ** adapters, |
| 3794 | size_t n_adapters, |
| 3795 | float * scales) { |
| 3796 | if (adapters == nullptr || scales == nullptr) { |
| 3797 | GGML_ASSERT(n_adapters == 0 && "invalid llama_set_adapters_lora call")if (!(n_adapters == 0 && "invalid llama_set_adapters_lora call" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3797, "GGML_ASSERT(%s) failed", "n_adapters == 0 && \"invalid llama_set_adapters_lora call\"" ); |
| 3798 | } |
| 3799 | |
| 3800 | ctx->set_adapters_lora(adapters, n_adapters, scales); |
| 3801 | |
| 3802 | return 0; |
| 3803 | } |
| 3804 | |
| 3805 | int32_t llama_set_adapter_cvec( |
| 3806 | llama_context * ctx, |
| 3807 | const float * data, |
| 3808 | size_t len, |
| 3809 | int32_t n_embd, |
| 3810 | int32_t il_start, |
| 3811 | int32_t il_end) { |
| 3812 | bool res = ctx->set_adapter_cvec(data, len, n_embd, il_start, il_end); |
| 3813 | |
| 3814 | return res ? 0 : -1; |
| 3815 | } |
| 3816 | |
| 3817 | // |
| 3818 | // memory |
| 3819 | // |
| 3820 | |
| 3821 | void llama_memory_clear(llama_memory_t mem, bool data) { |
| 3822 | if (!mem) { |
| 3823 | return; |
| 3824 | } |
| 3825 | |
| 3826 | mem->clear(data); |
| 3827 | } |
| 3828 | |
| 3829 | bool llama_memory_seq_rm( |
| 3830 | llama_memory_t mem, |
| 3831 | llama_seq_id seq_id, |
| 3832 | llama_pos p0, |
| 3833 | llama_pos p1) { |
| 3834 | if (!mem) { |
| 3835 | return true; |
| 3836 | } |
| 3837 | |
| 3838 | return mem->seq_rm(seq_id, p0, p1); |
| 3839 | } |
| 3840 | |
| 3841 | void llama_memory_seq_cp( |
| 3842 | llama_memory_t mem, |
| 3843 | llama_seq_id seq_id_src, |
| 3844 | llama_seq_id seq_id_dst, |
| 3845 | llama_pos p0, |
| 3846 | llama_pos p1) { |
| 3847 | if (!mem) { |
| 3848 | return; |
| 3849 | } |
| 3850 | |
| 3851 | mem->seq_cp(seq_id_src, seq_id_dst, p0, p1); |
| 3852 | } |
| 3853 | |
| 3854 | void llama_memory_seq_keep( |
| 3855 | llama_memory_t mem, |
| 3856 | llama_seq_id seq_id) { |
| 3857 | if (!mem) { |
| 3858 | return; |
| 3859 | } |
| 3860 | |
| 3861 | mem->seq_keep(seq_id); |
| 3862 | } |
| 3863 | |
| 3864 | void llama_memory_seq_add( |
| 3865 | llama_memory_t mem, |
| 3866 | llama_seq_id seq_id, |
| 3867 | llama_pos p0, |
| 3868 | llama_pos p1, |
| 3869 | llama_pos delta) { |
| 3870 | if (!mem) { |
| 3871 | return; |
| 3872 | } |
| 3873 | |
| 3874 | mem->seq_add(seq_id, p0, p1, delta); |
| 3875 | } |
| 3876 | |
| 3877 | void llama_memory_seq_div( |
| 3878 | llama_memory_t mem, |
| 3879 | llama_seq_id seq_id, |
| 3880 | llama_pos p0, |
| 3881 | llama_pos p1, |
| 3882 | int d) { |
| 3883 | if (!mem) { |
| 3884 | return; |
| 3885 | } |
| 3886 | |
| 3887 | mem->seq_div(seq_id, p0, p1, d); |
| 3888 | } |
| 3889 | |
| 3890 | llama_pos llama_memory_seq_pos_min( |
| 3891 | llama_memory_t mem, |
| 3892 | llama_seq_id seq_id) { |
| 3893 | if (!mem) { |
| 3894 | return -1; |
| 3895 | } |
| 3896 | |
| 3897 | return mem->seq_pos_min(seq_id); |
| 3898 | } |
| 3899 | |
| 3900 | llama_pos llama_memory_seq_pos_max( |
| 3901 | llama_memory_t mem, |
| 3902 | llama_seq_id seq_id) { |
| 3903 | if (!mem) { |
| 3904 | return -1; |
| 3905 | } |
| 3906 | |
| 3907 | return mem->seq_pos_max(seq_id); |
| 3908 | } |
| 3909 | |
| 3910 | bool llama_memory_can_shift(llama_memory_t mem) { |
| 3911 | if (!mem) { |
| 3912 | return false; |
| 3913 | } |
| 3914 | |
| 3915 | return mem->get_can_shift(); |
| 3916 | } |
| 3917 | |
| 3918 | // llama state API |
| 3919 | |
| 3920 | // deprecated |
| 3921 | size_t llama_get_state_size(llama_context * ctx) { |
| 3922 | return llama_state_get_size(ctx); |
| 3923 | } |
| 3924 | |
| 3925 | // deprecated |
| 3926 | size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) { |
| 3927 | return llama_state_get_data(ctx, dst, -1); |
| 3928 | } |
| 3929 | |
| 3930 | // deprecated |
| 3931 | size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) { |
| 3932 | return llama_state_set_data(ctx, src, -1); |
| 3933 | } |
| 3934 | |
| 3935 | // deprecated |
| 3936 | bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { |
| 3937 | return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out); |
| 3938 | } |
| 3939 | |
| 3940 | // deprecated |
| 3941 | bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { |
| 3942 | return llama_state_save_file(ctx, path_session, tokens, n_token_count); |
| 3943 | } |
| 3944 | |
| 3945 | // Returns the *actual* size of the state. |
| 3946 | // Intended to be used when saving to state to a buffer. |
| 3947 | size_t llama_state_get_size(llama_context * ctx) { |
| 3948 | return ctx->state_get_size(); |
| 3949 | } |
| 3950 | |
| 3951 | size_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) { |
| 3952 | ctx->synchronize(); |
| 3953 | |
| 3954 | return ctx->state_get_data(dst, size); |
| 3955 | } |
| 3956 | |
| 3957 | // Sets the state reading from the specified source address |
| 3958 | size_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) { |
| 3959 | ctx->synchronize(); |
| 3960 | |
| 3961 | return ctx->state_set_data(src, size); |
| 3962 | } |
| 3963 | |
| 3964 | bool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { |
| 3965 | ctx->synchronize(); |
| 3966 | |
| 3967 | tryif (true) { |
| 3968 | return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out); |
| 3969 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 3970 | LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading session file: %s\n" , __func__, err.what()); |
| 3971 | return false; |
| 3972 | } |
| 3973 | } |
| 3974 | |
| 3975 | bool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) { |
| 3976 | ctx->synchronize(); |
| 3977 | |
| 3978 | tryif (true) { |
| 3979 | return ctx->state_save_file(path_session, tokens, n_token_count); |
| 3980 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 3981 | LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving session file: %s\n" , __func__, err.what()); |
| 3982 | return false; |
| 3983 | } |
| 3984 | } |
| 3985 | |
| 3986 | size_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) { |
| 3987 | return llama_state_seq_get_size_ext(ctx, seq_id, 0); |
| 3988 | } |
| 3989 | |
| 3990 | size_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) { |
| 3991 | return llama_state_seq_get_data_ext(ctx, dst, size, seq_id, 0); |
| 3992 | } |
| 3993 | |
| 3994 | size_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) { |
| 3995 | return llama_state_seq_set_data_ext(ctx, src, size, seq_id, 0); |
| 3996 | } |
| 3997 | |
| 3998 | size_t llama_state_seq_get_size_ext(llama_context * ctx, llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 3999 | return ctx->state_seq_get_size(seq_id, flags); |
| 4000 | } |
| 4001 | |
| 4002 | size_t llama_state_seq_get_data_ext(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 4003 | ctx->synchronize(); |
| 4004 | |
| 4005 | return ctx->state_seq_get_data(seq_id, dst, size, flags); |
| 4006 | } |
| 4007 | size_t llama_state_seq_set_data_ext(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) { |
| 4008 | ctx->synchronize(); |
| 4009 | |
| 4010 | return ctx->state_seq_set_data(seq_id, src, size, flags); |
| 4011 | } |
| 4012 | |
| 4013 | size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) { |
| 4014 | ctx->synchronize(); |
| 4015 | |
| 4016 | tryif (true) { |
| 4017 | return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count); |
| 4018 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 4019 | LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving sequence state file: %s\n" , __func__, err.what()); |
| 4020 | return 0; |
| 4021 | } |
| 4022 | } |
| 4023 | |
| 4024 | size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) { |
| 4025 | ctx->synchronize(); |
| 4026 | |
| 4027 | tryif (true) { |
| 4028 | return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out); |
| 4029 | } catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) { |
| 4030 | LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading sequence state file: %s\n" , __func__, err.what()); |
| 4031 | return 0; |
| 4032 | } |
| 4033 | } |
| 4034 | |
| 4035 | /// |
| 4036 | |
| 4037 | int32_t llama_encode( |
| 4038 | llama_context * ctx, |
| 4039 | llama_batch batch) { |
| 4040 | const int ret = ctx->encode(batch); |
| 4041 | if (ret != 0) { |
| 4042 | LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to encode, ret = %d\n" , __func__, ret); |
| 4043 | } |
| 4044 | |
| 4045 | return ret; |
| 4046 | } |
| 4047 | |
| 4048 | int32_t llama_decode( |
| 4049 | llama_context * ctx, |
| 4050 | llama_batch batch) { |
| 4051 | const int ret = ctx->decode(batch); |
| 4052 | if (ret != 0 && ret != 1) { |
| 4053 | LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to decode, ret = %d\n" , __func__, ret); |
| 4054 | } |
| 4055 | |
| 4056 | return ret; |
| 4057 | } |
| 4058 | |
| 4059 | // |
| 4060 | // perf |
| 4061 | // |
| 4062 | |
| 4063 | llama_perf_context_data llama_perf_context(const llama_context * ctx) { |
| 4064 | llama_perf_context_data data = {}; |
| 4065 | |
| 4066 | if (ctx == nullptr) { |
| 4067 | return data; |
| 4068 | } |
| 4069 | |
| 4070 | data = ctx->perf_get_data(); |
| 4071 | |
| 4072 | return data; |
| 4073 | } |
| 4074 | |
| 4075 | void llama_perf_context_print(const llama_context * ctx) { |
| 4076 | const auto data = llama_perf_context(ctx); |
| 4077 | |
| 4078 | const double t_end_ms = 1e-3 * ggml_time_us(); |
| 4079 | |
| 4080 | LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: load time = %10.2f ms\n" , __func__, data.t_load_ms); |
| 4081 | LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval) |
| 4082 | __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval); |
| 4083 | LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data .n_eval, 1e3 / data.t_eval_ms * data.n_eval) |
| 4084 | __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data .n_eval, 1e3 / data.t_eval_ms * data.n_eval); |
| 4085 | LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: total time = %10.2f ms / %5d tokens\n" , __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data .n_eval)); |
| 4086 | LLAMA_LOG_INFO("%s: graphs reused = %10d\n", __func__, data.n_reused)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graphs reused = %10d\n" , __func__, data.n_reused); |
| 4087 | } |
| 4088 | |
| 4089 | void llama_perf_context_reset(llama_context * ctx) { |
| 4090 | ctx->perf_reset(); |
| 4091 | } |
| 4092 | |
| 4093 | // |
| 4094 | // training |
| 4095 | // |
| 4096 | |
| 4097 | bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata) { |
| 4098 | GGML_UNUSED(tensor)(void)(tensor); |
| 4099 | GGML_UNUSED(userdata)(void)(userdata); |
| 4100 | return true; |
| 4101 | } |
| 4102 | |
| 4103 | void llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) { |
| 4104 | ctx->opt_init(model, lopt_params); |
| 4105 | } |
| 4106 | |
| 4107 | void llama_opt_epoch( |
| 4108 | struct llama_context * ctx, |
| 4109 | ggml_opt_dataset_t dataset, |
| 4110 | ggml_opt_result_t result_train, |
| 4111 | ggml_opt_result_t result_eval, |
| 4112 | int64_t idata_split, |
| 4113 | ggml_opt_epoch_callback callback_train, |
| 4114 | ggml_opt_epoch_callback callback_eval) { |
| 4115 | ctx->opt_epoch( |
| 4116 | dataset, |
| 4117 | result_train, |
| 4118 | result_eval, |
| 4119 | idata_split, |
| 4120 | callback_train, |
| 4121 | callback_eval); |
| 4122 | } |
| 4123 | |
| 4124 | // |
| 4125 | // ext |
| 4126 | // |
| 4127 | |
| 4128 | llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx) { |
| 4129 | return ctx->memory_breakdown(); |
| 4130 | } |
| 4131 | |
| 4132 | llama_context * llama_get_ctx_other(struct llama_context * ctx) { |
| 4133 | return ctx->get_cparams().ctx_other; |
| 4134 | } |