/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp

Bug Summary

File:	root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp
Warning:	line 2187, column 9 Value stored to 'offset' is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -O2 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name llama-context.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=all -relaxed-aliasing -ffp-contract=off -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -target-feature +avx -target-feature +avx2 -target-feature +bmi2 -target-feature +f16c -target-feature +fma -target-feature +sse4.2 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -fcoverage-compilation-dir=/root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -resource-dir /usr/lib/llvm-23/lib/clang/23 -include /root/firefox-clang/config/gcc_hidden.h -include /root/firefox-clang/obj-x86_64-pc-linux-gnu/mozilla-config.h -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/stl_wrappers -D _GLIBCXX_ASSERTIONS=1 -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/system_wrappers -U _FORTIFY_SOURCE -D _FORTIFY_SOURCE=2 -D DEBUG=1 -D _GNU_SOURCE=1 -D GGML_USE_CPU=1 -D GGML_VERSION="GGML_VERSION" -D GGML_COMMIT="GGML_COMMIT" -D GGML_SHARED=1 -D LLAMA_SHARED=1 -D GGML_BUILD=1 -D LLAMA_BUILD=1 -D GGML_BACKEND_SHARED=1 -D GGML_BACKEND_BUILD=1 -D MOZ_HAS_MOZGLUE -I /root/firefox-clang/third_party/llama.cpp -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/third_party/llama.cpp -I /root/firefox-clang/third_party/llama.cpp/ggml -I /root/firefox-clang/third_party/llama.cpp/ggml/include -I /root/firefox-clang/third_party/llama.cpp/ggml/src -I /root/firefox-clang/third_party/llama.cpp/ggml/src/ggml-cpu -I /root/firefox-clang/third_party/llama.cpp/include -I /root/firefox-clang/third_party/llama.cpp/src -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nspr -I /root/firefox-clang/obj-x86_64-pc-linux-gnu/dist/include/nss -D MOZILLA_CLIENT -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/x86_64-linux-gnu/c++/16 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../include/c++/16/backward -internal-isystem /usr/lib/llvm-23/lib/clang/23/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/16/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -Wno-error=pessimizing-move -Wno-error=large-by-value-copy=128 -Wno-error=implicit-int-float-conversion -Wno-error=thread-safety-analysis -Wno-error=tautological-type-limit-compare -Wno-invalid-offsetof -Wno-range-loop-analysis -Wno-deprecated-anon-enum-enum-conversion -Wno-deprecated-enum-enum-conversion -Wno-inline-new-delete -Wno-error=deprecated-declarations -Wno-error=array-bounds -Wno-error=free-nonheap-object -Wno-error=atomic-alignment -Wno-error=deprecated-builtins -Wno-psabi -Wno-error=builtin-macro-redefined -Wno-vla-cxx-extension -Wno-unknown-warning-option -Wno-character-conversion -Wno-sign-compare -Wno-unused-function -Wno-tautological-unsigned-enum-zero-compare -Wno-implicit-fallthrough -Wno-unreachable-code -std=gnu++20 -fdeprecated-macro -ferror-limit 19 -fstrict-flex-arrays=1 -stack-protector 2 -fstack-clash-protection -ftrivial-auto-var-init=pattern -fno-rtti -fgnuc-version=4.2.1 -fno-implicit-modules -fskip-odr-check-in-gmf -fno-sized-deallocation -fno-aligned-allocation -fdiagnostics-absolute-paths -vectorize-loops -vectorize-slp -analyzer-checker optin.performance.Padding -analyzer-output=html -analyzer-config stable-report-filename=true -mllvm -dwarf-linkage-names=Abstract -faddrsig -fdwarf2-cfi-asm -o /tmp/scan-build-2026-09-01-224014-2642839-1 -x c++ /root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp

1	#include "llama-context.h"
2
3	#include "ggml.h"
4	#include "llama-arch.h"
5	#include "llama-graph.h"
6	#include "llama-impl.h"
7	#include "llama-batch.h"
8	#include "llama-io.h"
9	#include "llama-memory.h"
10	#include "llama-mmap.h"
11	#include "llama-model.h"
12	#include "llama-ext.h"
13	#include "llama.h"
14
15	#include <cinttypes>
16	#include <cmath>
17	#include <cstring>
18	#include <limits>
19	#include <stdexcept>
20
21	#include "moz-overrides.h"
22
23	//
24	// llama_context
25	//
26
27	static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
28	switch (ctx_type) {
29	case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;
30	case LLAMA_CONTEXT_TYPE_MTP : return LLM_GRAPH_TYPE_DECODER_MTP;
31	}
32	throwabort_with_suppression(); if (false) std::runtime_error("Unsupported ctx type");
33	}
34
35	llama_context::llama_context(
36	const llama_model & model,
37	llama_context_params params) :
38	model(model),
39	cvec(std::make_unique<llama_adapter_cvec>()),
40	loras(std::make_unique<llama_adapter_loras>()),
41	balloc(std::make_unique<llama_batch_allocr>(model.hparams.n_pos_per_embd())) {
42	// TODO warning when creating llama_context with awkward ctx size that is not a power of 2,
43	// may need to be backend-dependent
44	LLAMA_LOG_INFO("%s: constructing llama_context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: constructing llama_context\n" , __func__);
45
46	t_start_us = model.t_start_us;
47	t_load_us = model.t_load_us;
48
49	const auto & hparams = model.hparams;
50
51	cparams.n_seq_max = std::max(1u, params.n_seq_max);
52	if (cparams.n_seq_max > LLAMA_MAX_SEQ256) {
53	throwabort_with_suppression(); if (false) std::runtime_error("n_seq_max must be <= " + std::to_string(LLAMA_MAX_SEQ256));
54	}
55
56	cparams.n_rs_seq = params.n_rs_seq;
57	if (cparams.n_rs_seq > 0 && !llm_arch_supports_rs_rollback(model.arch)) {
58	LLAMA_LOG_DEBUG("%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n" , __func__, cparams.n_rs_seq)
59	__func__, cparams.n_rs_seq)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_rs_seq=%u requested but model arch does not support recurrent partial rollback; clamping to 0\n" , __func__, cparams.n_rs_seq);
60	cparams.n_rs_seq = 0;
61	}
62
63	cparams.n_threads = params.n_threads;
64	cparams.n_threads_batch = params.n_threads_batch;
65	cparams.yarn_ext_factor = params.yarn_ext_factor >= 0.0f ? params.yarn_ext_factor : hparams.yarn_ext_factor;
66	cparams.yarn_attn_factor = params.yarn_attn_factor >= 0.0f ? params.yarn_attn_factor : hparams.yarn_attn_factor;
67	cparams.yarn_beta_fast = params.yarn_beta_fast >= 0.0f ? params.yarn_beta_fast : hparams.yarn_beta_fast;
68	cparams.yarn_beta_slow = params.yarn_beta_slow >= 0.0f ? params.yarn_beta_slow : hparams.yarn_beta_slow;
69	cparams.embeddings = params.embeddings;
70	cparams.embeddings_nextn = false;
71	cparams.embeddings_nextn_masked = false;
72	cparams.offload_kqv = params.offload_kqv;
73	cparams.no_perf = params.no_perf;
74	cparams.warmup = false;
75
76	cparams.embeddings_layer_inp.resize(hparams.n_layer(), false);
77	embd_layer_inp.resize(hparams.n_layer());
78
79	cparams.ctx_type = params.ctx_type;
80	cparams.pooling_type = params.pooling_type;
81
82	cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
83	cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
84	cparams.rope_freq_scale = params.rope_freq_scale == 0.0f ? hparams.rope_freq_scale_train : params.rope_freq_scale;
85
86	cparams.n_ctx_orig_yarn = params.yarn_orig_ctx != 0 ? params.yarn_orig_ctx :
87	hparams.n_ctx_orig_yarn != 0 ? hparams.n_ctx_orig_yarn :
88	hparams.n_ctx_train;
89
90	cparams.cb_eval = params.cb_eval;
91	cparams.cb_eval_user_data = params.cb_eval_user_data;
92
93	cparams.ctx_other = nullptr;
94
95	// TODO: more generic
96	if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) {
97	if (params.ctx_other == nullptr) {
98	// TODO: change from runtime_error to llama_exception to avoid printing error message
99	throwabort_with_suppression(); if (false) std::runtime_error("Gemma4Assistant requires ctx_other to be set (this warning is normal during memory fitting)");
100	}
101
102	cparams.ctx_other = params.ctx_other;
103	}
104
105	if (model.arch == LLM_ARCH_EAGLE3) {
106	if (model.tok_embd == nullptr \|\| model.output == nullptr) {
107	if (params.ctx_other == nullptr) {
108	throwabort_with_suppression(); if (false) std::runtime_error("EAGLE3 requires ctx_other to be set (this warning is normal during memory fitting)");
109	}
110	cparams.ctx_other = params.ctx_other;
111	}
112	}
113
114	// Initialize backend samplers here so they are part of the sampling graph
115	// before the reserve passes run later in this function. This avoids a later
116	// re-reserve when graph nodes change.
117	if (params.samplers != nullptr && params.n_samplers > 0) {
118	for (size_t i = 0; i < params.n_samplers; ++i) {
119	const auto & config = params.samplers[i];
120
121	if (llama_sampler_chain_get(config.sampler, -1) == nullptr) {
122	throwabort_with_suppression(); if (false) std::runtime_error("the backend samplers must be of type llama_sampler_chain");
123	}
124
125	if (set_sampler(config.seq_id, config.sampler)) {
126	const int n_samplers = llama_sampler_chain_n(config.sampler);
127
128	LLAMA_LOG_INFO("%s: setting backend sampler for seq_id %d (n = %d)\n", __func__, config.seq_id, n_samplers)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: setting backend sampler for seq_id %d (n = %d)\n" , __func__, config.seq_id, n_samplers);
129	}
130	}
131	}
132
133	auto rope_scaling_type = params.rope_scaling_type;
134	if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
135	rope_scaling_type = hparams.rope_scaling_type_train;
136	}
137
138	if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
139	cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
140	}
141
142	if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
143	cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
144	}
145
146	if (cparams.yarn_ext_factor != 0) {
147	static auto get_mscale = [](float scale, float mscale) {
148	return scale <= 1.0f ? 1.0f : (0.1f * mscale * logf(scale) + 1.0f);
149	};
150
151	const float factor = 1.0f / cparams.rope_freq_scale;
152
153	// ref: https://github.com/huggingface/transformers/blob/6d00f6b0a5679c36510f203e4226e36f517c3032/src/transformers/modeling_rope_utils.py#L336-L348
154	if (hparams.rope_yarn_log_mul != 0.0f) {
155	// note: here we assume `mscale == 1.0f`
156	// TODO: start reading the actual value of mscale and handle the case where it is not 1.0f
157	float mscale = 1.0f;
158	const float mscale_all_dims = hparams.rope_yarn_log_mul;
159
160	// [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX]
161	// special-case DEEPSEEK v2:
162	// https://huggingface.co/deepseek-ai/DeepSeek-V2-Lite-Chat/blob/main/config.json#L42-L43
163	if (model.arch == LLM_ARCH_DEEPSEEK2 && mscale_all_dims != 1.0f) {
164	mscale = mscale_all_dims;
165	}
166
167	cparams.yarn_attn_factor = get_mscale(factor, mscale) / get_mscale(factor, mscale_all_dims);
168
169	LLAMA_LOG_WARN("%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n" , __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims )
170	__func__, cparams.yarn_attn_factor, mscale, mscale_all_dims)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: setting new yarn_attn_factor = %.4f (mscale == %.1f, mscale_all_dim = %.1f)\n" , __func__, cparams.yarn_attn_factor, mscale, mscale_all_dims );
171	} else {
172	cparams.yarn_attn_factor = get_mscale(factor, 1.0f);
173	}
174
175	// when YARN is applied with yarn_ext_factor != 0.0f, we need to cancel this factor:
176	// https://github.com/ggml-org/llama.cpp/blob/a81a569577cc38b32558958b048228150be63eae/ggml/src/ggml-cpu/ops.cpp#L5541-L5544
177	//
178	// ref: https://github.com/ggml-org/llama.cpp/discussions/7416
179	// https://github.com/ggml-org/llama.cpp/pull/17945
180	cparams.yarn_attn_factor = 1.0f / (1.0f + 0.1f logf(factor));
181	}
182
183	cparams.yarn_attn_factor *= hparams.rope_attn_factor;
184
185	if (cparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
186	if (hparams.pooling_type == LLAMA_POOLING_TYPE_UNSPECIFIED) {
187	cparams.pooling_type = LLAMA_POOLING_TYPE_NONE;
188	} else {
189	cparams.pooling_type = hparams.pooling_type;
190	}
191	}
192
193	if (params.attention_type == LLAMA_ATTENTION_TYPE_UNSPECIFIED) {
194	cparams.causal_attn = hparams.causal_attn;
195	} else {
196	cparams.causal_attn = params.attention_type == LLAMA_ATTENTION_TYPE_CAUSAL;
197	}
198
199	cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED;
200	cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO;
201
202	cparams.fused_gdn_ar = true;
203	cparams.fused_gdn_ch = true;
204	cparams.auto_fgdn = true;
205
206	// with causal attention, the batch size is limited by the context size
207	cparams.n_batch = cparams.causal_attn ? std::min(cparams.n_ctx, params.n_batch) : params.n_batch;
208
209	cparams.n_ubatch = std::min(cparams.n_batch, params.n_ubatch == 0 ? params.n_batch : params.n_ubatch);
210
211	cparams.n_outputs_max = params.n_outputs_max == 0 \|\| llama_model_has_encoder(&model) ? cparams.n_batch : params.n_outputs_max;
212
213	cparams.op_offload = params.op_offload;
214	cparams.kv_unified = params.kv_unified;
215
216	// initialized later
217	cparams.pipeline_parallel = false;
218
219	{
220	const char * LLAMA_GRAPH_REUSE_DISABLE = getenv("LLAMA_GRAPH_REUSE_DISABLE");
221	graph_reuse_disable = LLAMA_GRAPH_REUSE_DISABLE ? (atoi(LLAMA_GRAPH_REUSE_DISABLE) != 0) : graph_reuse_disable;
222
223	if (graph_reuse_disable) {
224	LLAMA_LOG_WARN("%s: graph reuse disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: graph reuse disabled\n" , __func__);
225	}
226	}
227
228	// ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732
229	cparams.n_ctx = GGML_PAD(cparams.n_ctx, 256)(((cparams.n_ctx) + (256) - 1) & ~((256) - 1));
230
231	if (cparams.kv_unified) {
232	cparams.n_ctx_seq = cparams.n_ctx;
233	} else {
234	cparams.n_ctx_seq = cparams.n_ctx / cparams.n_seq_max;
235	cparams.n_ctx_seq = GGML_PAD(cparams.n_ctx_seq, 256)(((cparams.n_ctx_seq) + (256) - 1) & ~((256) - 1));
236
237	if (cparams.n_ctx_seq == 0) {
238	throwabort_with_suppression(); if (false) std::runtime_error("n_ctx_seq == 0");
239	}
240
241	if (cparams.n_ctx != cparams.n_ctx_seq * cparams.n_seq_max) {
242	cparams.n_ctx = cparams.n_ctx_seq * cparams.n_seq_max;
243	LLAMA_LOG_WARN("%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n", __func__, cparams.n_ctx)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx is not divisible by n_seq_max - rounding down to %u\n" , __func__, cparams.n_ctx);
244	}
245	}
246
247	LLAMA_LOG_INFO("%s: n_seq_max = %u\n", __func__, cparams.n_seq_max)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_seq_max = %u\n" , __func__, cparams.n_seq_max);
248	LLAMA_LOG_INFO("%s: n_ctx = %u\n", __func__, cparams.n_ctx)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ctx = %u\n" , __func__, cparams.n_ctx);
249	LLAMA_LOG_INFO("%s: n_ctx_seq = %u\n", __func__, cparams.n_ctx_seq)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ctx_seq = %u\n" , __func__, cparams.n_ctx_seq);
250	LLAMA_LOG_INFO("%s: n_batch = %u\n", __func__, cparams.n_batch)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_batch = %u\n" , __func__, cparams.n_batch);
251	LLAMA_LOG_INFO("%s: n_ubatch = %u\n", __func__, cparams.n_ubatch)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_ubatch = %u\n" , __func__, cparams.n_ubatch);
252	LLAMA_LOG_INFO("%s: causal_attn = %d\n", __func__, cparams.causal_attn)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: causal_attn = %d\n" , __func__, cparams.causal_attn);
253	LLAMA_LOG_INFO("%s: flash_attn = %s\n", __func__, llama_flash_attn_type_name(params.flash_attn_type))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: flash_attn = %s\n" , __func__, llama_flash_attn_type_name(params.flash_attn_type ));
254	LLAMA_LOG_INFO("%s: kv_unified = %s\n", __func__, cparams.kv_unified ? "true" : "false")llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: kv_unified = %s\n" , __func__, cparams.kv_unified ? "true" : "false");
255	LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: freq_base = %.1f\n" , __func__, cparams.rope_freq_base);
256	LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: freq_scale = %g\n" , __func__, cparams.rope_freq_scale);
257	LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_rs_seq = %u\n" , __func__, cparams.n_rs_seq);
258	LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: n_outputs_max = %u\n" , __func__, cparams.n_outputs_max);
259
260	if (cparams.n_ctx_seq < hparams.n_ctx_train) {
261	LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train)
262	__func__, cparams.n_ctx_seq, hparams.n_ctx_train)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train);
263	}
264
265	if (cparams.n_ctx_seq > hparams.n_ctx_train) {
266	LLAMA_LOG_WARN("%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train)
267	__func__, cparams.n_ctx_seq, hparams.n_ctx_train)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: n_ctx_seq (%u) > n_ctx_train (%u) -- possible training context overflow\n" , __func__, cparams.n_ctx_seq, hparams.n_ctx_train);
268	}
269
270	if (!hparams.vocab_only) {
271	// GPU backends
272	for (const auto & dev : model.devices) {
273	ggml_backend_t backend = ggml_backend_dev_init(dev.dev, nullptr);
274	if (backend == nullptr) {
275	throwabort_with_suppression(); if (false) std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev.dev)));
276	}
277	backends.emplace_back(backend);
278	}
279
280	// add ACCEL backends (such as BLAS)
281	for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
282	ggml_backend_dev_t dev = ggml_backend_dev_get(i);
283	if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {
284	ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);
285	if (backend == nullptr) {
286	throwabort_with_suppression(); if (false) std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev)));
287	}
288	backends.emplace_back(backend);
289	}
290	}
291
292	// add CPU backend
293	backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
294	if (backend_cpu == nullptr) {
295	throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize CPU backend");
296	}
297	backends.emplace_back(backend_cpu);
298
299	// create a list of the set_n_threads functions in the backends
300	for (auto & backend : backends) {
301	ggml_backend_dev_t dev = ggml_backend_get_device(backend.get());
302	ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr;
303	if (reg) {
304	auto ggml_backend_set_n_threads_fn = (ggml_backend_set_n_threads_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_n_threads");
305	if (ggml_backend_set_n_threads_fn) {
306	set_n_threads_fns.emplace_back(backend.get(), ggml_backend_set_n_threads_fn);
307	}
308	}
309	}
310
311	llama_set_abort_callback(this, params.abort_callback, params.abort_callback_data);
312
313	// graph outputs buffer
314	{
315	if (output_reserve(params.n_seq_max) < params.n_seq_max) {
316	throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve initial output buffer");
317	}
318
319	LLAMA_LOG_INFO("%s: %10s output buffer size = %8.2f MiB\n", __func__,llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0)
320	ggml_backend_buffer_name (buf_output.get()),llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0)
321	ggml_backend_buffer_get_size(buf_output.get()) / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s output buffer size = %8.2f MiB\n" , __func__, ggml_backend_buffer_name (buf_output.get()), ggml_backend_buffer_get_size (buf_output.get()) / 1024.0 / 1024.0);
322	}
323	}
324
325	// init the memory module
326	if (!hparams.vocab_only) {
327	llama_memory_params params_mem = {
328	/.type_k =/ params.type_k,
329	/.type_v =/ params.type_v,
330	/.swa_full =/ params.swa_full,
331	/.ctx_type =/ cparams.ctx_type,
332	/.mem_other =/ llama_get_memory(cparams.ctx_other),
333	};
334
335	memory.reset(model.create_memory(params_mem, cparams));
336	}
337
338	// init backends
339	if (!hparams.vocab_only) {
340	LLAMA_LOG_DEBUG("%s: enumerating backends\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: enumerating backends\n" , __func__);
341
342	backend_buft.clear();
343	backend_ptrs.clear();
344	backend_buf_exp_size.clear();
345
346	for (auto & backend : backends) {
347	auto * buft = ggml_backend_get_default_buffer_type(backend.get());
348	auto backend_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
349
350	if (backend_type == GGML_BACKEND_DEVICE_TYPE_CPU && !model.devices.empty()) {
351	// use the host buffer of the first device CPU for faster transfer of the intermediate state
352	const auto & dev = model.devices[0];
353	auto * host_buft = ggml_backend_dev_host_buffer_type(dev.dev);
354	if (host_buft) {
355	buft = host_buft;
356	}
357	}
358
359	backend_buft.push_back(buft);
360	backend_ptrs.push_back(backend.get());
361	backend_buf_exp_size.push_back(0);
362	}
363
364	LLAMA_LOG_DEBUG("%s: backend_ptrs.size() = %zu\n", __func__, backend_ptrs.size())llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: backend_ptrs.size() = %zu\n" , __func__, backend_ptrs.size());
365
366	// TODO: move these checks to ggml_backend_sched
367	// enabling pipeline parallelism in the scheduler increases memory usage, so it is only done when necessary
368	bool pipeline_parallel =
369	model.n_devices() > 1 &&
370	model.n_gpu_layers() > model.hparams.n_layer_all &&
371	model.split_mode() == LLAMA_SPLIT_MODE_LAYER &&
372	cparams.offload_kqv &&
373	!model.has_tensor_overrides();
374
375	// pipeline parallelism requires support for async compute and events in all devices
376	if (pipeline_parallel) {
377	for (auto & backend : backends) {
378	auto dev_type = ggml_backend_dev_type(ggml_backend_get_device(backend.get()));
379	if (dev_type == GGML_BACKEND_DEVICE_TYPE_CPU) {
380	// ignore CPU backend
381	// TODO: should we ignore ACCEL types too?
382	continue;
383	}
384	auto * dev = ggml_backend_get_device(backend.get());
385	ggml_backend_dev_props props;
386	ggml_backend_dev_get_props(dev, &props);
387	if (!props.caps.async \|\| !props.caps.events) {
388	// device does not support async compute or events
389	pipeline_parallel = false;
390	break;
391	}
392	}
393	}
394
395	cparams.pipeline_parallel = pipeline_parallel;
396
397	if (cparams.pipeline_parallel) {
398	LLAMA_LOG_INFO("%s: pipeline parallelism enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: pipeline parallelism enabled\n" , __func__);
399	}
400
401	sched_reserve();
402
403	if (!cparams.flash_attn) {
404	if (ggml_is_quantized(params.type_v)) {
405	throwabort_with_suppression(); if (false) std::runtime_error("quantized V cache was requested, but this requires Flash Attention");
406	}
407	}
408	}
409
410	// Initialize the full vocabulary token ids for backend samplers.
411	{
412	const int n_vocab = model.vocab.n_tokens();
413
414	sampling.token_ids_full_vocab.resize(n_vocab);
415	for (int i = 0; i < n_vocab; ++i) {
416	sampling.token_ids_full_vocab[i] = i;
417	}
418	}
419	}
420
421	llama_context::~llama_context() {
422	if (!model.hparams.no_alloc) {
423	for (size_t i = 0; i < backend_ptrs.size(); ++i) {
424	ggml_backend_t backend = backend_ptrs[i];
425	ggml_backend_buffer_type_t buft = backend_buft[i];
426
427	const size_t size_exp = backend_buf_exp_size[i];
428	const size_t size_act = ggml_backend_sched_get_buffer_size(sched.get(), backend);
429	if (size_exp == size_act) {
430	LLAMA_LOG_DEBUG("%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n",llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0))
431	__func__, ggml_backend_buft_name(buft), size_act / (1024.01024.0), size_exp / (1024.01024.0))llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: %10s compute buffer size is %8.4f MiB, matches expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0));
432	} else {
433	LLAMA_LOG_WARN("%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0))
434	__func__, ggml_backend_buft_name(buft), size_act / (1024.01024.0), size_exp / (1024.01024.0))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: %10s compute buffer size of %8.4f MiB, does not match expectation of %8.4f MiB\n" , __func__, ggml_backend_buft_name(buft), size_act / (1024.0* 1024.0), size_exp / (1024.0*1024.0));
435	}
436	}
437	}
438	ggml_opt_free(opt_ctx);
439	}
440
441	void llama_context::sched_reserve() {
442	if (!sched_need_reserve) {
443	return;
444	}
445
446	sched_need_reserve = false;
447
448	LLAMA_LOG_INFO("%s: reserving ...\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserving ...\n" , __func__);
449
450	synchronize();
451
452	const int64_t t_start_us = ggml_time_us();
453
454	const uint32_t n_seqs = cparams.n_seq_max;
455	const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
456
457	const size_t max_nodes = this->graph_max_nodes(n_tokens);
458
459	LLAMA_LOG_DEBUG("%s: max_nodes = %zu\n", __func__, max_nodes)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: max_nodes = %zu\n" , __func__, max_nodes);
460
461	gf_res_prev.reset(new llm_graph_result(max_nodes));
462	gf_res_reserve.reset(new llm_graph_result(max_nodes));
463
464	sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));
465
466	llama_memory_context_ptr mctx;
467	if (memory) {
468	LLAMA_LOG_DEBUG("%s: reserving full memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reserving full memory module\n" , __func__);
469	mctx = memory->init_full();
470	if (!mctx) {
471	throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize memory module");
472	}
473	}
474
475	// avoid reserving graphs with zero outputs - assume one output per sequence
476	const int n_outputs = n_seqs;
477
478	LLAMA_LOG_DEBUG("%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: worst-case: n_tokens = %d, n_seqs = %d, n_outputs = %d\n" , __func__, n_tokens, n_seqs, n_outputs);
479
480	// resolve automatic Flash Attention use
481	if (cparams.auto_fa) {
482	auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
483	if (!gf) {
484	throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for Flash Attention check");
485	}
486
487	const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FATTN"__fattn__") + 1;
488	bool fa_device_mismatch = false;
489	for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
490	ggml_tensor * n = ggml_graph_node(gf, i);
491	if (n->op != GGML_OP_FLASH_ATTN_EXT) {
492	continue;
493	}
494	ggml_backend_dev_t device_fa = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
495
496	// TODO: instead of the tensor names, use a map to keep track of which (FA) tensors belong to which layer
497	GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FATTN "-", prefix_len) == 0)if (!(strncmp(n->name, "__fattn__" "-", prefix_len) == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 497, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FATTN \"-\", prefix_len) == 0" );
498	const int il = std::stoi(n->name + prefix_len);
499	ggml_backend_dev_t device_kv = model.dev_layer(il);
500	if (device_fa != device_kv) {
501	LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the Flash Attention tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa))
502	"is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa))
503	__func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_fa))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the Flash Attention tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_fa));
504	// FIXME: fa_device_mismatch logic is wrong for --no-kv-offload, but this is broken anyways
505	fa_device_mismatch = true;
506	break;
507	}
508	}
509
510	if (fa_device_mismatch) {
511	cparams.flash_attn = false;
512	LLAMA_LOG_WARN("%s: Flash Attention was auto, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: Flash Attention was auto, set to disabled\n" , __func__);
513	} else {
514	cparams.flash_attn = true;
515	LLAMA_LOG_INFO("%s: Flash Attention was auto, set to enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: Flash Attention was auto, set to enabled\n" , __func__);
516	}
517
518	cparams.auto_fa = false;
519	}
520
521	if (cparams.auto_fgdn) {
522	LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: resolving fused Gated Delta Net support:\n" , __func__);
523
524	if (cparams.fused_gdn_ar) {
525	auto * gf = graph_reserve(1, n_seqs, n_outputs, mctx.get(), true);
526	if (!gf) {
527	throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for fused Gated Delta Net check (autoregressive)");
528	}
529
530	const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_AR"__fgdn_ar__") + 1;
531	bool gdn_device_mismatch = false;
532	for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
533	ggml_tensor * n = ggml_graph_node(gf, i);
534	if (n->op != GGML_OP_GATED_DELTA_NET) {
535	continue;
536	}
537	ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
538
539	GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR "-", prefix_len) == 0)if (!(strncmp(n->name, "__fgdn_ar__" "-", prefix_len) == 0 )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 539, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_AR \"-\", prefix_len) == 0" );
540	const int il = std::stoi(n->name + prefix_len);
541	ggml_backend_dev_t device_kv = model.dev_layer(il);
542	if (device_gdn != device_kv) {
543	LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn))
544	"is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn))
545	__func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn));
546	gdn_device_mismatch = true;
547	break;
548	}
549	}
550
551	if (gdn_device_mismatch) {
552	cparams.fused_gdn_ar = false;
553	LLAMA_LOG_WARN("%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: fused Gated Delta Net (autoregressive) not supported, set to disabled\n" , __func__);
554	} else {
555	LLAMA_LOG_INFO("%s: fused Gated Delta Net (autoregressive) enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: fused Gated Delta Net (autoregressive) enabled\n" , __func__);
556	}
557	}
558
559	if (cparams.fused_gdn_ch) {
560	// more than one token in the batch per sequence in order to take the chunked path
561	// note: n_outputs must match n_tokens for embedding models with mean/rank pooling,
562	// because build_pooling creates inp_mean with shape [n_tokens, n_seqs] and multiplies
563	// it with t_embd which is reduced to [n_outputs, ...] via out_ids. if n_outputs != n_tokens,
564	// the ggml_mul_mat assertion fails.
565	const uint32_t n_tokens_ch = 16*n_seqs;
566	auto * gf = graph_reserve(n_tokens_ch, n_seqs, n_tokens_ch, mctx.get(), true);
567	if (!gf) {
568	throwabort_with_suppression(); if (false) std::runtime_error("failed to reserve graph for fused Gated Delta Net check (chunked)");
569	}
570
571	const size_t prefix_len = strlen(LLAMA_TENSOR_NAME_FGDN_CH"__fgdn_ch__") + 1;
572	bool gdn_device_mismatch = false;
573	for (int i = 0; i < ggml_graph_n_nodes(gf); i++) {
574	ggml_tensor * n = ggml_graph_node(gf, i);
575	if (n->op != GGML_OP_GATED_DELTA_NET) {
576	continue;
577	}
578	ggml_backend_dev_t device_gdn = ggml_backend_get_device(ggml_backend_sched_get_tensor_backend(sched.get(), n));
579
580	GGML_ASSERT(strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH "-", prefix_len) == 0)if (!(strncmp(n->name, "__fgdn_ch__" "-", prefix_len) == 0 )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 580, "GGML_ASSERT(%s) failed", "strncmp(n->name, LLAMA_TENSOR_NAME_FGDN_CH \"-\", prefix_len) == 0" );
581	const int il = std::stoi(n->name + prefix_len);
582	ggml_backend_dev_t device_kv = model.dev_layer(il);
583	if (device_gdn != device_kv) {
584	LLAMA_LOG_WARN("%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor "llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn))
585	"is assigned to device %s (usually due to missing support)\n",llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn))
586	__func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name(device_gdn))llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: layer %d is assigned to device %s but the fused Gated Delta Net tensor " "is assigned to device %s (usually due to missing support)\n" , __func__, il, ggml_backend_dev_name(device_kv), ggml_backend_dev_name (device_gdn));
587	gdn_device_mismatch = true;
588	break;
589	}
590	}
591
592	if (gdn_device_mismatch) {
593	cparams.fused_gdn_ch = false;
594	LLAMA_LOG_WARN("%s: fused Gated Delta Net (chunked) not supported, set to disabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: fused Gated Delta Net (chunked) not supported, set to disabled\n" , __func__);
595	} else {
596	LLAMA_LOG_INFO("%s: fused Gated Delta Net (chunked) enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: fused Gated Delta Net (chunked) enabled\n" , __func__);
597	}
598	}
599
600	cparams.auto_fgdn = false;
601	}
602
603	// reserve worst-case graph
604	int n_splits_pp = -1;
605	int n_nodes_pp = -1;
606
607	int n_splits_tg = -1;
608	int n_nodes_tg = -1;
609
610	const uint32_t n_outputs_pp = std::min(n_tokens, cparams.n_outputs_max);
611
612	// reserve pp (prompt processing) graph first so that buffers are only allocated once
613	{
614	auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(),
615	model.hparams.no_alloc, model.hparams.no_alloc ? backend_buf_exp_size.data() : nullptr);
616	if (!gf) {
617	if (cparams.pipeline_parallel) {
618	LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: compute buffer allocation failed, retrying without pipeline parallelism\n" , __func__);
619	cparams.pipeline_parallel = false;
620	sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
621	gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
622	}
623	if (!gf) {
624	throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute pp buffers");
625	}
626	}
627
628	n_splits_pp = ggml_backend_sched_get_n_splits(sched.get());
629	n_nodes_pp = ggml_graph_n_nodes(gf);
630	}
631
632	// reserve with tg (token generation) graph to get the number of splits and nodes
633	{
634	auto * gf = graph_reserve(n_seqs, n_seqs, n_seqs, mctx.get(), model.hparams.no_alloc);
635	if (!gf) {
636	throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute tg buffers");
637	}
638
639	n_splits_tg = ggml_backend_sched_get_n_splits(sched.get());
640	n_nodes_tg = ggml_graph_n_nodes(gf);
641	}
642
643	// reserve again with pp graph to avoid ggml-alloc reallocations during inference
644	{
645	// TODO: not sure if the following graph would be worst case for multi-stream KV caches:
646	//
647	// auto * gf = graph_reserve(n_tokens, 1, n_tokens, mctx.get());
648	//
649	auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get(), model.hparams.no_alloc);
650	if (!gf) {
651	throwabort_with_suppression(); if (false) std::runtime_error("failed to allocate compute pp buffers");
652	}
653	}
654
655	for (size_t i = 0; i < backend_ptrs.size(); ++i) {
656	ggml_backend_t backend = backend_ptrs[i];
657	ggml_backend_buffer_type_t buft = backend_buft[i];
658	if (!model.hparams.no_alloc) {
659	backend_buf_exp_size[i] = ggml_backend_sched_get_buffer_size(sched.get(), backend);
660	}
661	if (backend_buf_exp_size[i] > 1) {
662	LLAMA_LOG_INFO("%s: %10s compute buffer size = %8.2f MiB\n", __func__,llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0)
663	ggml_backend_buft_name(buft),llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0)
664	backend_buf_exp_size[i] / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: %10s compute buffer size = %8.2f MiB\n" , __func__, ggml_backend_buft_name(buft), backend_buf_exp_size [i] / 1024.0 / 1024.0);
665	}
666	}
667
668	if (n_nodes_pp == n_nodes_tg) {
669	LLAMA_LOG_INFO("%s: graph nodes = %d\n", __func__, n_nodes_pp)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph nodes = %d\n" , __func__, n_nodes_pp);
670	} else {
671	LLAMA_LOG_INFO("%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n", __func__, n_nodes_pp, n_tokens, n_nodes_tg)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph nodes = %d (with bs=%d), %d (with bs=1)\n" , __func__, n_nodes_pp, n_tokens, n_nodes_tg);
672	}
673
674	if (n_splits_pp == n_splits_tg) {
675	LLAMA_LOG_INFO("%s: graph splits = %d\n", __func__, n_splits_pp)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph splits = %d\n" , __func__, n_splits_pp);
676	} else {
677	LLAMA_LOG_INFO("%s: graph splits = %d (with bs=%d), %d (with bs=1)\n", __func__, n_splits_pp, n_tokens, n_splits_tg)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graph splits = %d (with bs=%d), %d (with bs=1)\n" , __func__, n_splits_pp, n_tokens, n_splits_tg);
678	}
679
680	const int64_t t_end_us = ggml_time_us();
681
682	LLAMA_LOG_INFO("%s: reserve took %.2f ms, sched copies = %d\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserve took %.2f ms, sched copies = %d\n" , __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies (sched.get()))
683	__func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get()))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: reserve took %.2f ms, sched copies = %d\n" , __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies (sched.get()));
684	}
685
686	void llama_context::synchronize() {
687	if (!sched) {
688	return;
689	}
690
691	ggml_backend_sched_synchronize(sched.get());
692
693	// FIXME: if multiple single tokens are evaluated without a synchronization,
694	// the stats will be added to the prompt evaluation stats
695	// this should only happen when using batch size 1 to evaluate a batch
696
697	// add the evaluation to the stats
698	if (n_queued_tokens == 1) {
699	if (!cparams.no_perf) {
700	t_eval_us += ggml_time_us() - t_compute_start_us;
701	}
702	n_eval++;
703	} else if (n_queued_tokens > 1) {
704	if (!cparams.no_perf) {
705	t_p_eval_us += ggml_time_us() - t_compute_start_us;
706	}
707	n_p_eval += n_queued_tokens;
708	}
709
710	// get a more accurate load time, upon first eval
711	if (n_queued_tokens > 0 && !has_evaluated_once) {
712	t_load_us = ggml_time_us() - t_start_us;
713	has_evaluated_once = true;
714	}
715
716	n_queued_tokens = 0;
717	t_compute_start_us = 0;
718	}
719
720	const llama_model & llama_context::get_model() const {
721	return model;
722	}
723
724	const llama_cparams & llama_context::get_cparams() const {
725	return cparams;
726	}
727
728	ggml_backend_sched_t llama_context::get_sched() const {
729	return sched.get();
730	}
731
732	uint32_t llama_context::n_ctx() const {
733	return cparams.n_ctx;
734	}
735
736	uint32_t llama_context::n_ctx_seq() const {
737	return cparams.n_ctx_seq;
738	}
739
740	uint32_t llama_context::n_batch() const {
741	return cparams.n_batch;
742	}
743
744	uint32_t llama_context::n_ubatch() const {
745	return cparams.n_ubatch;
746	}
747
748	uint32_t llama_context::n_seq_max() const {
749	return cparams.n_seq_max;
750	}
751
752	uint32_t llama_context::n_threads() const {
753	return cparams.n_threads;
754	}
755
756	uint32_t llama_context::n_threads_batch() const {
757	return cparams.n_threads_batch;
758	}
759
760	llama_memory_t llama_context::get_memory() const {
761	return memory.get();
762	}
763
764	bool llama_context::memory_update(bool optimize) {
765	if (!memory) {
766	return false;
767	}
768
769	{
770	const auto mctx = memory->init_update(this, optimize);
771	switch (mctx->get_status()) {
772	case LLAMA_MEMORY_STATUS_SUCCESS:
773	{
774	// noop
775	} break;
776	case LLAMA_MEMORY_STATUS_NO_UPDATE:
777	{
778	// no updates need to be performed
779	return false;
780	}
781	case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
782	case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
783	{
784	LLAMA_LOG_ERROR("%s: failed to prepare memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to prepare memory update\n" , __func__);
785	return false;
786	}
787	}
788
789	// reset the previous graph result to make sure that it won't be reused
790	// TODO: change the mctx->apply() to return information if a graph reserve is needed
791	// reset the graph result only if the memory module did reset the scheduler
792	gf_res_prev->reset();
793
794	if (!mctx->apply()) {
795	LLAMA_LOG_ERROR("%s: failed to apply memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to apply memory update\n" , __func__);
796	}
797	}
798
799	// if the memory module did any computation, we have to reserve a new worst-case graph
800	{
801	const auto mctx = memory->init_full();
802	if (!mctx) {
803	throwabort_with_suppression(); if (false) std::runtime_error("failed to initialize memory context");
804	}
805
806	const uint32_t n_seqs = cparams.n_seq_max;
807	const uint32_t n_tokens = std::min(cparams.n_ctx, cparams.n_ubatch);
808
809	const uint32_t n_outputs_max = std::min(n_tokens, cparams.n_outputs_max);
810
811	auto * gf = graph_reserve(n_tokens, n_seqs, n_outputs_max, mctx.get());
812	if (!gf) {
813	LLAMA_LOG_ERROR("%s: failed to reserve graph after the memory update\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to reserve graph after the memory update\n" , __func__);
814	}
815	}
816
817	return true;
818	}
819
820	enum llama_pooling_type llama_context::pooling_type() const {
821	return cparams.pooling_type;
822	}
823
824	float * llama_context::get_logits() {
825	output_reorder();
826
827	return logits.data;
828	}
829
830	int64_t llama_context::output_resolve_row(int32_t i) const {
831	int64_t j = -1;
832
833	// support negative indices (last output row)
834	if (i < 0) {
835	j = n_outputs + i;
836	if (j < 0) {
837	throwabort_with_suppression(); if (false) std::runtime_error(format("negative index out of range [0, %d)", n_outputs));
838	}
839	} else if ((size_t) i >= output_ids.size()) {
840	throwabort_with_suppression(); if (false) std::runtime_error(format("out of range [0, %zu)", output_ids.size()));
841	} else {
842	// use output_ids to translate the batch token index into a row number
843	// that holds this token's data.
844	j = output_ids[i];
845	}
846
847	if (j < 0) {
848	// the batch token was not configured to output anything
849	throwabort_with_suppression(); if (false) std::runtime_error(format("batch.logits[%d] != true", i));
850	}
851
852	if (j >= n_outputs) {
853	throwabort_with_suppression(); if (false) std::runtime_error(format("corrupt output buffer (j=%" PRId64"l" "d" ", n_outputs=%d)", j, n_outputs));
854	}
855
856	return j;
857	}
858
859	float * llama_context::get_logits_ith(int32_t i) {
860	output_reorder();
861
862	tryif (true) {
863	if (logits.data == nullptr) {
864	throwabort_with_suppression(); if (false) std::runtime_error("no logits");
865	}
866
867	const int64_t j = output_resolve_row(i);
868	return logits.data + j*model.vocab.n_tokens();
869	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
870	LLAMA_LOG_ERROR("%s: invalid logits id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid logits id %d, reason: %s\n" , __func__, i, err.what());
871	#ifndef NDEBUG
872	GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 872, "fatal error");
873	#else
874	return nullptr;
875	#endif
876	}
877	}
878
879	float * llama_context::get_embeddings() {
880	output_reorder();
881
882	return embd.data;
883	}
884
885	llama_token * llama_context::get_sampled_tokens() const{
886	return sampling.sampled.data;
887	}
888
889	float * llama_context::get_embeddings_ith(int32_t i) {
890	output_reorder();
891
892	tryif (true) {
893	if (embd.data == nullptr) {
894	throwabort_with_suppression(); if (false) std::runtime_error("no embeddings");
895	}
896
897	const int64_t j = output_resolve_row(i);
898	const uint32_t n_embd_out = model.hparams.n_embd_out();
899	return embd.data + j*n_embd_out;
900	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
901	LLAMA_LOG_ERROR("%s: invalid embeddings id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid embeddings id %d, reason: %s\n" , __func__, i, err.what());
902	#ifndef NDEBUG
903	GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 903, "fatal error");
904	#else
905	return nullptr;
906	#endif
907	}
908	}
909
910	float * llama_context::get_embeddings_seq(llama_seq_id seq_id) {
911	auto it = embd_seq.find(seq_id);
912	if (it == embd_seq.end()) {
913	return nullptr;
914	}
915
916	return it->second.data();
917	}
918
919	float * llama_context::get_embeddings_nextn() {
920	output_reorder();
921
922	return embd_nextn.data;
923	}
924
925	float * llama_context::get_embeddings_nextn_ith(int32_t i) {
926	output_reorder();
927
928	tryif (true) {
929	if (embd_nextn.data == nullptr) {
930	throwabort_with_suppression(); if (false) std::runtime_error("no nextn embeddings");
931	}
932
933	const uint32_t n_embd = model.hparams.n_embd_out();
934
935	if (!cparams.embeddings_nextn_masked) {
936	// unmasked: nextn rows are stored densely, indexed by raw token position.
937	if (i < 0 \|\| (size_t)(i + 1) * n_embd > embd_nextn.size) {
938	throwabort_with_suppression(); if (false) std::runtime_error(format("out of range [0, %zu)", embd_nextn.size / n_embd));
939	}
940	return embd_nextn.data + (size_t) i * n_embd;
941	}
942
943	const int64_t j = output_resolve_row(i);
944	return embd_nextn.data + j*n_embd;
945	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
946	LLAMA_LOG_ERROR("%s: invalid nextn embeddings id %d, reason: %s\n", __func__, i, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid nextn embeddings id %d, reason: %s\n" , __func__, i, err.what());
947	#ifndef NDEBUG
948	GGML_ABORT("fatal error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 948, "fatal error");
949	#else
950	return nullptr;
951	#endif
952	}
953	}
954
955	float * llama_context::get_embeddings_layer_inp(uint32_t lid) {
956	output_reorder();
957
958	GGML_ASSERT(lid < embd_layer_inp.size() && embd_layer_inp[lid].has_data())if (!(lid < embd_layer_inp.size() && embd_layer_inp [lid].has_data())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 958, "GGML_ASSERT(%s) failed", "lid < embd_layer_inp.size() && embd_layer_inp[lid].has_data()" );
959
960	return embd_layer_inp[lid].data;
961	}
962
963	llama_token llama_context::get_sampled_token_ith(int32_t idx) {
964	output_reorder();
965
966	if (!sampling.sampled.has_data()) {
967	return LLAMA_TOKEN_NULL-1;
968	}
969
970	tryif (true) {
971	const int64_t row = output_resolve_row(idx);
972	GGML_ASSERT(row < (int64_t) sampling.sampled.size)if (!(row < (int64_t) sampling.sampled.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 972, "GGML_ASSERT(%s) failed", "row < (int64_t) sampling.sampled.size" );
973	return sampling.sampled.data[row];
974	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
975	LLAMA_LOG_ERROR("%s: invalid backend sampled token id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled token id %d, reason: %s\n" , __func__, idx, err.what());
976	return LLAMA_TOKEN_NULL-1;
977	}
978	}
979
980	float * llama_context::get_sampled_probs_ith(int32_t idx) {
981	output_reorder();
982
983	if (!sampling.probs.has_data()) {
984	return nullptr;
985	}
986
987	tryif (true) {
988	const int64_t row = output_resolve_row(idx);
989	if ((size_t) row >= sampling.probs_count.size() \|\| sampling.probs_count[row] == 0) {
990	return nullptr;
991	}
992	return sampling.probs.data + row*model.vocab.n_tokens();
993	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
994	LLAMA_LOG_ERROR("%s: invalid backend sampled probs id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled probs id %d, reason: %s\n" , __func__, idx, err.what());
995	return nullptr;
996	}
997	}
998
999	float * llama_context::get_sampled_logits_ith(int32_t idx) {
1000	output_reorder();
1001
1002	if (!sampling.logits.has_data()) {
1003	return nullptr;
1004	}
1005
1006	tryif (true) {
1007	const int64_t row = output_resolve_row(idx);
1008	if ((size_t) row >= sampling.logits_count.size() \|\| sampling.logits_count[row] == 0) {
1009	return nullptr;
1010	}
1011	return sampling.logits.data + row*model.vocab.n_tokens();
1012	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
1013	LLAMA_LOG_ERROR("%s: invalid backend sampled logits id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled logits id %d, reason: %s\n" , __func__, idx, err.what());
1014	return nullptr;
1015	}
1016	}
1017
1018	const llama_token * llama_context::get_sampled_candidates_ith(int32_t idx) {
1019	output_reorder();
1020
1021	tryif (true) {
1022	const int64_t row = output_resolve_row(idx);
1023	if (sampling.candidates.has_data() &&
1024	(size_t) row < sampling.candidates_count.size() &&
1025	sampling.candidates_count[row] > 0) {
1026	return sampling.candidates.data + row*model.vocab.n_tokens();
1027	}
1028	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
1029	// fallback to full vocab list
1030	GGML_UNUSED(err)(void)(err);
1031	}
1032
1033	return sampling.token_ids_full_vocab.data();
1034	}
1035
1036	size_t llama_context::get_sampled_candidates_count(int32_t idx) {
1037	output_reorder();
1038
1039	if (!sampling.candidates.has_data()) {
1040	return 0;
1041	}
1042
1043	tryif (true) {
1044	const int64_t row = output_resolve_row(idx);
1045	if ((size_t) row >= sampling.candidates_count.size()) {
1046	return 0;
1047	}
1048	return sampling.candidates_count[row];
1049	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
1050	LLAMA_LOG_ERROR("%s: invalid backend sampled candidates count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled candidates count id %d, reason: %s\n" , __func__, idx, err.what());
1051	return 0;
1052	}
1053	}
1054
1055	size_t llama_context::get_sampled_logits_count(int32_t idx) {
1056	output_reorder();
1057
1058	if (!sampling.logits.has_data()) {
1059	return model.vocab.n_tokens();
1060	}
1061
1062	tryif (true) {
1063	const int64_t row = output_resolve_row(idx);
1064	if ((size_t) row >= sampling.logits_count.size()) {
1065	return 0;
1066	}
1067	return sampling.logits_count[row];
1068	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
1069	LLAMA_LOG_ERROR("%s: invalid backend sampled logits count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled logits count id %d, reason: %s\n" , __func__, idx, err.what());
1070	return 0;
1071	}
1072	}
1073
1074	size_t llama_context::get_sampled_probs_count(int32_t idx) {
1075	output_reorder();
1076
1077	if (!sampling.probs.has_data()) {
1078	return 0;
1079	}
1080
1081	tryif (true) {
1082	const int64_t row = output_resolve_row(idx);
1083	if ((size_t) row >= sampling.probs_count.size()) {
1084	return 0;
1085	}
1086	return sampling.probs_count[row];
1087	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
1088	LLAMA_LOG_ERROR("%s: invalid backend sampled probs count id %d, reason: %s\n", __func__, idx, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: invalid backend sampled probs count id %d, reason: %s\n" , __func__, idx, err.what());
1089	return 0;
1090	}
1091	}
1092
1093
1094	void llama_context::attach_threadpool(
1095	ggml_threadpool_t threadpool,
1096	ggml_threadpool_t threadpool_batch) {
1097	LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ );
1098
1099	this->threadpool = threadpool;
1100	this->threadpool_batch = threadpool_batch ? threadpool_batch : threadpool;
1101	}
1102
1103	void llama_context::detach_threadpool() {
1104	LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ );
1105
1106	this->threadpool = nullptr;
1107	this->threadpool_batch = nullptr;
1108	}
1109
1110	void llama_context::set_n_threads(int32_t n_threads, int32_t n_threads_batch) {
1111	LLAMA_LOG_DEBUG("%s: n_threads = %d, n_threads_batch = %d\n", __func__, n_threads, n_threads_batch)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: n_threads = %d, n_threads_batch = %d\n" , __func__, n_threads, n_threads_batch);
1112
1113	cparams.n_threads = n_threads;
1114	cparams.n_threads_batch = n_threads_batch;
1115	}
1116
1117	void llama_context::set_abort_callback(bool (abort_callback)(void data), void * abort_callback_data) {
1118	LLAMA_LOG_DEBUG("%s: call\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: call\n", __func__ );
1119
1120	this->abort_callback = abort_callback;
1121	this->abort_callback_data = abort_callback_data;
1122
1123	for (auto & backend : backends) {
1124	auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend.get()));
1125	if (reg) {
1126	auto * set_abort_callback_fn = (ggml_backend_set_abort_callback_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_set_abort_callback");
1127	if (set_abort_callback_fn) {
1128	set_abort_callback_fn(backend.get(), this->abort_callback, this->abort_callback_data);
1129	}
1130	}
1131	}
1132	}
1133
1134	void llama_context::set_embeddings(bool value) {
1135	LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value);
1136
1137	cparams.embeddings = value;
1138
1139	// TODO: not sure yet if we want to reserve here
1140	//sched_need_reserve = true;
1141	}
1142
1143	void llama_context::set_embeddings_nextn(bool value, bool masked) {
1144	LLAMA_LOG_DEBUG("%s: value = %d, masked = %d\n", __func__, value, masked)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d, masked = %d\n" , __func__, value, masked);
1145
1146	cparams.embeddings_nextn = value;
1147	cparams.embeddings_nextn_masked = masked;
1148	}
1149
1150	void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) {
1151	LLAMA_LOG_DEBUG("%s: lid = %d, enable = %d\n", __func__, lid, enable)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: lid = %d, enable = %d\n" , __func__, lid, enable);
1152
1153	GGML_ASSERT(lid < model.hparams.n_layer())if (!(lid < model.hparams.n_layer())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1153, "GGML_ASSERT(%s) failed", "lid < model.hparams.n_layer()" );
1154
1155	cparams.embeddings_layer_inp[lid] = enable;
1156
1157	// note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected
1158	sched_need_reserve = true;
1159	}
1160
1161	void llama_context::set_causal_attn(bool value) {
1162	LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value);
1163
1164	if (cparams.causal_attn == value) {
1165	return;
1166	}
1167
1168	cparams.causal_attn = value;
1169
1170	sched_need_reserve = true;
1171	}
1172
1173	void llama_context::set_warmup(bool value) {
1174	LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: value = %d\n", __func__ , value);
1175
1176	if (cparams.warmup == value) {
1177	return;
1178	}
1179
1180	cparams.warmup = value;
1181
1182	// warmups are usually with small batches, so no need to reserve
1183	//sched_need_reserve = true;
1184	}
1185
1186	bool llama_context::set_sampler(llama_seq_id seq_id, llama_sampler * sampler) {
1187	if (!sampler && sampling.samplers.count(seq_id) == 0) {
1188	return true;
1189	}
1190
1191	LLAMA_LOG_DEBUG("%s: seq_id = %d, sampler = %p\n", __func__, (int) seq_id, (void ) sampler)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: seq_id = %d, sampler = %p\n" , __func__, (int) seq_id, (void ) sampler);
1192
1193	if (sampler && model.split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
1194	static bool warned = false;
1195	if (!warned) {
1196	LLAMA_LOG_WARN("%s: backend sampling not supported with SPLIT_MODE_TENSOR; using CPU\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: backend sampling not supported with SPLIT_MODE_TENSOR; using CPU\n" , __func__);
1197	warned = true;
1198	}
1199	if (sampling.samplers.count(seq_id) > 0) {
1200	sched_need_reserve = true;
1201	}
1202	sampling.samplers.erase(seq_id);
1203	return false;
1204	}
1205
1206	const bool can_offload =
1207	sampler &&
1208	sampler->iface->backend_init &&
1209	sampler->iface->backend_apply &&
1210	llama_sampler_chain_n(sampler) > 0;
1211
1212	if (sampler && can_offload) {
1213	auto * buft = ggml_backend_dev_buffer_type(model.dev_output());
1214
1215	sampler->iface->backend_init(sampler, buft);
1216
1217	sampling.samplers[seq_id] = sampler;
1218
1219	sched_need_reserve = true;
1220
1221	return true;
1222	}
1223
1224	if (sampler && !can_offload) {
1225	LLAMA_LOG_WARN("%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\n", __func__, llama_sampler_name(sampler), seq_id)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: sampler '%s' for seq_id = %d, cannot be offloaded to the backend\n" , __func__, llama_sampler_name(sampler), seq_id);
1226
1227	if (sampling.samplers.count(seq_id) > 0) {
1228	sched_need_reserve = true;
1229	}
1230
1231	sampling.samplers.erase(seq_id);
1232
1233	return false;
1234	}
1235
1236	sampling.samplers.erase(seq_id);
1237
1238	sched_need_reserve = true;
1239
1240	return true;
1241	}
1242
1243	void llama_context::set_adapters_lora(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {
1244	LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void ) adapters)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: adapters = %p\n" , __func__, (void ) adapters);
1245
1246	if (adapters_lora_are_same(adapters, n_adapters, scales)) {
1247	return;
1248	}
1249
1250	loras.reset(new llama_adapter_loras());
1251
1252	for (size_t i = 0; i < n_adapters; i ++) {
1253	if (scales[i] != 0.0f) {
1254	loras->insert({adapters[i], scales[i]});
1255	}
1256	}
1257
1258	sched_need_reserve = true;
1259	}
1260
1261	bool llama_context::adapters_lora_are_same(llama_adapter_lora ** adapters, size_t n_adapters, float * scales) {
1262	LLAMA_LOG_DEBUG("%s: adapters = %p\n", __func__, (void ) adapters)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: adapters = %p\n" , __func__, (void ) adapters);
1263
1264	// Adapters with a zero scale are never added to `loras`, so also ignore them for the comparison.
1265	size_t n_non_zero = 0;
1266
1267	for (size_t i = 0; i < n_adapters; i ++) {
1268	if (scales[i] == 0.0f) {
1269	continue;
1270	}
1271	n_non_zero++;
1272
1273	auto it = loras->find(adapters[i]);
1274
1275	if (it == loras->end() \|\| it->second != scales[i]) {
1276	return false;
1277	}
1278	}
1279
1280	if (n_non_zero != loras->size()) {
1281	return false;
1282	}
1283
1284	return true;
1285	}
1286
1287	bool llama_context::set_adapter_cvec(
1288	const float * data,
1289	size_t len,
1290	int32_t n_embd,
1291	int32_t il_start,
1292	int32_t il_end) {
1293	LLAMA_LOG_DEBUG("%s: il_start = %d, il_end = %d\n", __func__, il_start, il_end)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: il_start = %d, il_end = %d\n" , __func__, il_start, il_end);
1294
1295	bool res = cvec->apply(model, data, len, n_embd, il_start, il_end);
1296
1297	sched_need_reserve = true;
1298
1299	return res;
1300	}
1301
1302	llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, llm_graph_type gtype, llama_memory_context_i * mctx, ggml_status & ret) {
1303	if (mctx && !mctx->apply()) {
1304	LLAMA_LOG_ERROR("%s: failed to apply memory context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to apply memory context\n" , __func__);
1305	ret = GGML_STATUS_FAILED;
1306	return nullptr;
1307	}
1308
1309	auto * res = gf_res_prev.get();
1310	auto * gf = res->get_gf();
1311
1312	// the new graph parameters
1313	// in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters
1314	const auto gparams = graph_params(res, ubatch, mctx, gtype);
1315
1316	if (!graph_reuse_disable && res->can_reuse(gparams)) {
1317	//LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__);
1318
1319	// with pipeline parallelism, the previous graph_compute_async may still be running
1320	// on the GPU. we must synchronize before set_inputs to avoid overwriting input tensors
1321	// that the previous compute is still reading.
1322	if (cparams.pipeline_parallel) {
1323	ggml_backend_sched_synchronize(sched.get());
1324	}
1325
1326	n_reused++;
1327	} else {
1328	res->reset();
1329
1330	ggml_backend_sched_reset(sched.get());
1331	ggml_backend_sched_set_eval_callback(sched.get(), cparams.cb_eval, cparams.cb_eval_user_data);
1332
1333	//const auto t_start_us = ggml_time_us();
1334
1335	gf = model.build_graph(gparams);
1336
1337	//LLAMA_LOG_INFO("graph build time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0);
1338
1339	if (!gf) {
1340	LLAMA_LOG_ERROR("%s: failed to initialize graph\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize graph\n" , __func__);
1341	ret = GGML_STATUS_FAILED;
1342	return nullptr;
1343	}
1344
1345	if (!ggml_backend_sched_alloc_graph(sched.get(), gf)) {
1346	LLAMA_LOG_ERROR("%s: failed to allocate graph\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate graph\n" , __func__);
1347	ret = GGML_STATUS_ALLOC_FAILED;
1348	return nullptr;
1349	}
1350	}
1351
1352	// set the input data for the input tensors
1353	{
1354	//const auto t_start_us = ggml_time_us();
1355
1356	// FIXME this call causes a crash if any model inputs were not used in the graph and were therefore not allocated
1357	res->set_inputs(&ubatch);
1358
1359	//LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0);
1360	}
1361
1362	const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1);
1363	if (status != GGML_STATUS_SUCCESS) {
1364	LLAMA_LOG_ERROR("%s: failed to compute graph, compute status: %d\n", __func__, status)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to compute graph, compute status: %d\n" , __func__, status);
1365	ret = status;
1366	return nullptr;
1367	}
1368
1369	ret = GGML_STATUS_SUCCESS;
1370
1371	return res;
1372	}
1373
1374	int llama_context::encode(const llama_batch & batch_inp) {
1375	// MTP hook batches carry both token (next-token id) and embd (h_nextn row),
1376	// so accept either present rather than requiring exactly one.
1377	GGML_ASSERT(batch_inp.token \|\| batch_inp.embd)if (!(batch_inp.token \|\| batch_inp.embd)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1377, "GGML_ASSERT(%s) failed", "batch_inp.token \|\| batch_inp.embd" );
1378
1379	if (batch_inp.n_tokens == 0) {
1380	LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_tokens == 0\n" , __func__);
1381	return -1;
1382	}
1383
1384	const auto & hparams = model.hparams;
1385
1386	// eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
1387	const int64_t n_embd = hparams.n_embd_inp();
1388	const int64_t n_vocab = model.vocab.n_tokens();
1389
1390	// note: during encode, we always pass the full sequence starting from pos = 0
1391	if (!balloc->init(batch_inp, model.vocab, nullptr, n_embd, cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max, true)) {
1392	LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__);
1393	return -1;
1394	}
1395
1396	const uint32_t n_tokens = balloc->get_n_tokens();
1397
1398	// [TAG_NO_CACHE_PAD]
1399	// TODO: add new split mode where we pad the input sequences so that ubatch.equal_seqs == true
1400	const llama_ubatch ubatch = balloc->split_simple(n_tokens);
1401
1402	// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
1403	GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens")if (!(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1403, "GGML_ASSERT(%s) failed", "cparams.n_ubatch >= n_tokens && \"encoder requires n_ubatch >= n_tokens\"" );
1404
1405	if (t_compute_start_us == 0) {
1406	t_compute_start_us = ggml_time_us();
1407	}
1408
1409	// TODO: this clear of the buffer can easily be forgotten - need something better
1410	embd_seq.clear();
1411
1412	sched_reserve();
1413
1414	n_queued_tokens += n_tokens;
1415
1416	// reserve output buffer
1417	if (output_reserve(n_tokens) < n_tokens) {
1418	LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %u outputs\n" , __func__, n_tokens);
1419	return -2;
1420	};
1421
1422	for (uint32_t i = 0; i < n_tokens; ++i) {
1423	output_ids[i] = i;
1424	}
1425
1426	n_outputs = n_tokens;
1427
1428	const auto causal_attn_org = cparams.causal_attn;
1429
1430	// always use non-causal attention for encoder graphs
1431	// TODO: this is a tmp solution until we have a proper way to support enc-dec models
1432	// ref: https://github.com/ggml-org/llama.cpp/pull/12181#issuecomment-2730451223
1433	cparams.causal_attn = false;
1434
1435	ggml_status status;
1436	const auto * res = process_ubatch(ubatch, LLM_GRAPH_TYPE_ENCODER, nullptr, status);
1437
1438	cparams.causal_attn = causal_attn_org;
1439
1440	if (!res) {
1441	switch (status) {
1442	case GGML_STATUS_ABORTED: return 2;
1443	case GGML_STATUS_ALLOC_FAILED: return -2;
1444	case GGML_STATUS_FAILED: return -3;
1445	case GGML_STATUS_SUCCESS: GGML_ABORT("should not happen")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1445, "should not happen");
1446	}
1447	}
1448
1449	auto * t_logits = res->get_logits();
1450	auto * t_embd = res->get_embd_pooled() ? res->get_embd_pooled() : res->get_embd();
1451	auto * t_h_nextn = cparams.embeddings_nextn ? res->get_h_nextn() : nullptr;
1452
1453	// extract logits
1454	if (logits.data && t_logits) {
1455	ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);
1456	GGML_ASSERT(backend_res != nullptr)if (!(backend_res != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1456, "GGML_ASSERT(%s) failed", "backend_res != nullptr");
1457	GGML_ASSERT(logits.data != nullptr)if (!(logits.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1457, "GGML_ASSERT(%s) failed", "logits.data != nullptr");
1458
1459	ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokensn_vocabsizeof(float));
1460	}
1461
1462	// extract embeddings
1463	if (embd.data && t_embd) {
1464	ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
1465	GGML_ASSERT(backend_embd != nullptr)if (!(backend_embd != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1465, "GGML_ASSERT(%s) failed", "backend_embd != nullptr");
1466
1467	switch (cparams.pooling_type) {
1468	case LLAMA_POOLING_TYPE_NONE:
1469	{
1470	// extract token embeddings
1471	GGML_ASSERT(embd.data != nullptr)if (!(embd.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1471, "GGML_ASSERT(%s) failed", "embd.data != nullptr");
1472	const uint32_t n_embd_out = hparams.n_embd_out();
1473
1474	GGML_ASSERT(n_tokensn_embd_out <= (int64_t) embd.size)if (!(n_tokensn_embd_out <= (int64_t) embd.size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1474, "GGML_ASSERT(%s) failed", "n_tokens*n_embd_out <= (int64_t) embd.size" );
1475	ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokensn_embd_outsizeof(float));
1476	} break;
1477	case LLAMA_POOLING_TYPE_MEAN:
1478	case LLAMA_POOLING_TYPE_CLS:
1479	case LLAMA_POOLING_TYPE_LAST:
1480	{
1481	// extract sequence embeddings
1482	auto & embd_seq_out = embd_seq;
1483
1484	for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
1485	const llama_seq_id seq_id = ubatch.seq_id_unq[s];
1486	const int32_t seq_idx = ubatch.seq_idx[seq_id];
1487
1488	// use n_embd_out (not n_embd_inp) - the pooled embedding has the model's
1489	// output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl)
1490	const uint32_t n_embd_out = hparams.n_embd_out();
1491	embd_seq_out[seq_id].resize(n_embd_out);
1492	ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_outseq_idx)sizeof(float), n_embd_out*sizeof(float));
1493	}
1494	} break;
1495	case LLAMA_POOLING_TYPE_RANK:
1496	{
1497	// extract the rerank score - n_cls_out floats per sequence
1498	auto & embd_seq_out = embd_seq;
1499
1500	const uint32_t n_cls_out = hparams.n_cls_out;
1501
1502	for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
1503	const llama_seq_id seq_id = ubatch.seq_id_unq[s];
1504	const int32_t seq_idx = ubatch.seq_idx[seq_id];
1505
1506	embd_seq_out[seq_id].resize(n_cls_out);
1507	ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_outseq_idx)sizeof(float), n_cls_out*sizeof(float));
1508	}
1509	} break;
1510	case LLAMA_POOLING_TYPE_UNSPECIFIED:
1511	{
1512	GGML_ABORT("unknown pooling type")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1512, "unknown pooling type");
1513	}
1514	}
1515	}
1516
1517	// extract nextn embeddings (hidden state before the final output norm)
1518	if (embd_nextn.data && t_h_nextn && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) {
1519	ggml_backend_t backend_h = ggml_backend_sched_get_tensor_backend(sched.get(), t_h_nextn);
1520	GGML_ASSERT(backend_h != nullptr)if (!(backend_h != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1520, "GGML_ASSERT(%s) failed", "backend_h != nullptr");
1521
1522	const uint32_t n_embd = hparams.n_embd_out();
1523	GGML_ASSERT(n_tokensn_embd <= (int64_t) embd_nextn.size)if (!(n_tokensn_embd <= (int64_t) embd_nextn.size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1523, "GGML_ASSERT(%s) failed", "n_tokens*n_embd <= (int64_t) embd_nextn.size" );
1524	ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokensn_embdsizeof(float));
1525	}
1526
1527	// TODO: hacky solution
1528	if (model.arch == LLM_ARCH_T5 && t_embd) {
1529	//cross.t_embd = t_embd;
1530
1531	synchronize();
1532
1533	cross.n_embd = t_embd->ne[0];
1534	cross.n_enc = t_embd->ne[1];
1535	cross.v_embd.resize(cross.n_embd*cross.n_enc);
1536	memcpy(cross.v_embd.data(), embd.data, ggml_nbytes(t_embd));
1537
1538	const auto & batch = balloc->get_batch();
1539
1540	// remember the sequence ids used during the encoding - needed for cross attention later
1541	cross.seq_ids_enc.resize(n_tokens);
1542	for (uint32_t i = 0; i < n_tokens; i++) {
1543	cross.seq_ids_enc[i].clear();
1544
1545	for (int s = 0; s < batch.n_seq_id[i]; s++) {
1546	const llama_seq_id seq_id = batch.seq_id[i][s];
1547
1548	cross.seq_ids_enc[i].insert(seq_id);
1549	}
1550	}
1551	}
1552
1553	return 0;
1554	}
1555
1556	static std::map<llama_seq_id, uint32_t> build_seq_to_output_row(const llama_ubatch & ubatch, uint32_t row_offset) {
1557	std::map<llama_seq_id, uint32_t> seq_to_row;
1558	// how many output tokens we have seen so far for this ubatch.
1559	uint32_t local = 0;
1560	for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
1561	// skip tokens that are not output.
1562	if (!ubatch.output[i]) {
1563	continue;
1564	}
1565
1566	const llama_seq_id seq_id = ubatch.seq_id[i][0];
1567	// row_offset is the number of output tokens before this ubatch.
1568	seq_to_row[seq_id] = row_offset + local;
1569	++local;
1570	}
1571	return seq_to_row;
1572	}
1573
1574	static void copy_tensor_async_ints(
1575	const std::map<llama_seq_id, ggml_tensor*> & tensor_map,
1576	const buffer_view<llama_token> & sampled,
1577	const std::map<llama_seq_id, uint32_t> & seq_to_row,
1578	ggml_backend_sched_t sched) {
1579	if (!sampled.has_data()) {
1580	return;
1581	}
1582
1583	for (const auto & [seq_id, tensor] : tensor_map) {
1584	auto it = seq_to_row.find(seq_id);
1585	if (it == seq_to_row.end()) {
1586	continue;
1587	}
1588
1589	const uint32_t row = it->second;
1590	GGML_ASSERT(row < sampled.size)if (!(row < sampled.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1590, "GGML_ASSERT(%s) failed", "row < sampled.size");
1591
1592	GGML_ASSERT(ggml_is_contiguous(tensor) && "sampled tokens tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "sampled tokens tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1592, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"sampled tokens tensor must be contiguous for async copy\"" );
1593
1594	ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);
1595	ggml_backend_tensor_get_async(backend, tensor, sampled.data + row, 0, sizeof(sampled.data[row]));
1596	}
1597	}
1598
1599	static void copy_tensor_async_floats(
1600	const std::map<llama_seq_id, ggml_tensor*> & tensor_map,
1601	const buffer_view<float> & dst,
1602	size_t stride,
1603	std::vector<uint32_t> & counts,
1604	const std::map<llama_seq_id, uint32_t> & seq_to_row,
1605	ggml_backend_sched_t sched) {
1606	if (!dst.has_data()) {
1607	return;
1608	}
1609
1610	for (const auto & [seq_id, tensor] : tensor_map) {
1611	auto it = seq_to_row.find(seq_id);
1612	if (it == seq_to_row.end()) {
1613	continue;
1614	}
1615
1616	const uint32_t row = it->second;
1617	GGML_ASSERT(row < counts.size())if (!(row < counts.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1617, "GGML_ASSERT(%s) failed", "row < counts.size()");
1618
1619	GGML_ASSERT(ggml_is_contiguous(tensor) && "logits/probs tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "logits/probs tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1619, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"logits/probs tensor must be contiguous for async copy\"" );
1620
1621	ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);
1622	float * row_ptr = dst.data + (size_t) row * stride;
1623	ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor));
1624
1625	// Update the actual number of logits/probabilities that were written for this row.
1626	counts[row] = ggml_nelements(tensor);
1627	}
1628	}
1629
1630	static void copy_tensor_async_candidates(
1631	const std::map<llama_seq_id, ggml_tensor*> & tensor_map,
1632	const buffer_view<llama_token> & dst,
1633	size_t stride,
1634	std::vector<uint32_t> & counts,
1635	const std::map<llama_seq_id, uint32_t> & seq_to_row,
1636	ggml_backend_sched_t sched) {
1637	if (!dst.has_data()) {
1638	return;
1639	}
1640
1641	for (const auto & [seq_id, tensor] : tensor_map) {
1642	auto it = seq_to_row.find(seq_id);
1643	if (it == seq_to_row.end()) {
1644	continue;
1645	}
1646
1647	const uint32_t row = it->second;
1648	GGML_ASSERT(row < counts.size())if (!(row < counts.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1648, "GGML_ASSERT(%s) failed", "row < counts.size()");
1649
1650	GGML_ASSERT(ggml_is_contiguous(tensor) && "candidates tensor must be contiguous for async copy")if (!(ggml_is_contiguous(tensor) && "candidates tensor must be contiguous for async copy" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1650, "GGML_ASSERT(%s) failed", "ggml_is_contiguous(tensor) && \"candidates tensor must be contiguous for async copy\"" );
1651
1652	ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched, tensor);
1653	llama_token * row_ptr = dst.data + (size_t) row * stride;
1654	ggml_backend_tensor_get_async(backend, tensor, row_ptr, 0, ggml_nbytes(tensor));
1655
1656	// Update the actual number of candidates that were written.
1657	counts[row] = ggml_nelements(tensor);
1658	}
1659	}
1660
1661	static bool needs_raw_logits(const llama_ubatch & ubatch, const std::map<llama_seq_id, llama_sampler *> & samplers) {
1662	for (uint32_t i = 0; i < ubatch.n_tokens; i++) {
1663	if (!ubatch.output[i]) {
1664	continue;
1665	}
1666
1667	// Check if the output token has at least one sequence without a backend sampler.
1668	for (int32_t j = 0; j < ubatch.n_seq_id[i]; ++j) {
1669	llama_seq_id seq_id = ubatch.seq_id[i][j];
1670	if (samplers.find(seq_id) == samplers.end()) {
1671	return true;
1672	}
1673	}
1674	}
1675	return false; // all sequences use backend sampling
1676	}
1677
1678	int llama_context::decode(const llama_batch & batch_inp) {
1679	// MTP hook batches carry both token (next-token id) and embd (h_nextn row),
1680	// so accept either present rather than requiring exactly one.
1681	GGML_ASSERT(batch_inp.token \|\| batch_inp.embd)if (!(batch_inp.token \|\| batch_inp.embd)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1681, "GGML_ASSERT(%s) failed", "batch_inp.token \|\| batch_inp.embd" );
1682
1683	if (!memory) {
1684	LLAMA_LOG_DEBUG("%s: cannot decode batches with this context (calling encode() instead)\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: cannot decode batches with this context (calling encode() instead)\n" , __func__);
1685	return encode(batch_inp);
1686	}
1687
1688	if (batch_inp.n_tokens == 0) {
1689	LLAMA_LOG_ERROR("%s: n_tokens == 0\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_tokens == 0\n" , __func__);
1690	return -1;
1691	}
1692
1693	const auto & vocab = model.vocab;
1694	const auto & hparams = model.hparams;
1695
1696	const int64_t n_vocab = vocab.n_tokens();
1697	const int64_t n_embd = hparams.n_embd_inp();
1698
1699	// when computing embeddings, all tokens are output
1700	const bool output_all = cparams.embeddings;
1701	const bool has_samplers = !sampling.samplers.empty();
1702
1703	const uint32_t n_seq_max = cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max;
1704
1705	// TODO: avoid this workaround in the future
1706	if (has_samplers && batch_inp.logits) {
1707	std::vector<int32_t> seq_output_count(n_seq_max, 0);
1708
1709	for (int32_t i = 0; i < batch_inp.n_tokens; ++i) {
1710	if (batch_inp.logits[i] == 0) {
1711	continue;
1712	}
1713
1714	const int ns = batch_inp.n_seq_id ? batch_inp.n_seq_id[i] : 1;
1715
1716	for (int32_t s = 0; s < ns; ++s) {
1717	const llama_seq_id seq_id = batch_inp.seq_id ? batch_inp.seq_id[i][s] : 0;
1718
1719	seq_output_count[seq_id]++;
1720	if (seq_output_count[seq_id] > 1) {
1721	LLAMA_LOG_ERROR("%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n" , __func__, seq_id, seq_output_count[seq_id])
1722	__func__, seq_id, seq_output_count[seq_id])llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: backend sampling requires at most one output token per sequence (seq_id %d had %d)\n" , __func__, seq_id, seq_output_count[seq_id]);
1723	return -1;
1724	}
1725	}
1726	}
1727	}
1728
1729	if (!balloc->init(batch_inp, vocab, memory.get(), n_embd, n_seq_max, output_all)) {
1730	LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__);
1731	return -1;
1732	}
1733
1734	const uint32_t n_tokens_all = balloc->get_n_tokens();
1735	const uint32_t n_outputs_all = balloc->get_n_outputs();
1736
1737	if (output_all) {
1738	// require that all tokens are output
1739	if (n_outputs_all != n_tokens_all) {
1740	LLAMA_LOG_ERROR("%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n" , __func__, n_outputs_all, n_tokens_all)
1741	__func__, n_outputs_all, n_tokens_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: pooled embedding requires that all tokens are output (n_outputs_all = %d, n_tokens_all = %d)\n" , __func__, n_outputs_all, n_tokens_all);
1742	return -1;
1743	}
1744	}
1745
1746	GGML_ASSERT(n_tokens_all <= cparams.n_batch)if (!(n_tokens_all <= cparams.n_batch)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1746, "GGML_ASSERT(%s) failed", "n_tokens_all <= cparams.n_batch" );
1747
1748	GGML_ASSERT((cparams.causal_attn \|\| cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens")if (!((cparams.causal_attn \|\| cparams.n_ubatch >= n_tokens_all ) && "non-causal attention requires n_ubatch >= n_tokens" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1748, "GGML_ASSERT(%s) failed", "(cparams.causal_attn \|\| cparams.n_ubatch >= n_tokens_all) && \"non-causal attention requires n_ubatch >= n_tokens\"" );
1749
1750	if (t_compute_start_us == 0) {
1751	t_compute_start_us = ggml_time_us();
1752	}
1753	n_queued_tokens += n_tokens_all;
1754
1755	// TODO: this clear of the buffer can easily be forgotten - need something better
1756	embd_seq.clear();
1757	output_swaps.clear();
1758
1759	sched_reserve();
1760
1761	bool did_optimize = false;
1762
1763	// handle any pending shifts/copies
1764	memory_update(false);
1765
1766	llama_memory_context_ptr mctx;
1767
1768	while (true) {
1769	mctx = memory->init_batch(*balloc, cparams.n_ubatch, output_all);
1770	if (!mctx) {
1771	return -2;
1772	}
1773
1774	switch (mctx->get_status()) {
1775	case LLAMA_MEMORY_STATUS_SUCCESS:
1776	{
1777	} break;
1778	case LLAMA_MEMORY_STATUS_NO_UPDATE:
1779	{
1780	LLAMA_LOG_ERROR("%s: unexpected memory context status: %d\n", __func__, mctx->get_status())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unexpected memory context status: %d\n" , __func__, mctx->get_status());
1781
1782	return -2;
1783	}
1784	case LLAMA_MEMORY_STATUS_FAILED_PREPARE:
1785	{
1786	if (!did_optimize) {
1787	did_optimize = true;
1788
1789	if (memory_update(true)) {
1790	LLAMA_LOG_DEBUG("%s: retrying batch size %d after cache optimization\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: retrying batch size %d after cache optimization\n" , __func__, balloc->get_n_tokens());
1791
1792	continue;
1793	}
1794	}
1795
1796	LLAMA_LOG_WARN("%s: failed to find a memory slot for batch of size %d\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: failed to find a memory slot for batch of size %d\n" , __func__, balloc->get_n_tokens());
1797
1798	return 1;
1799	}
1800	case LLAMA_MEMORY_STATUS_FAILED_COMPUTE:
1801	{
1802	LLAMA_LOG_ERROR("%s: compute failed while preparing batch of size %d\n", __func__, balloc->get_n_tokens())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: compute failed while preparing batch of size %d\n" , __func__, balloc->get_n_tokens());
1803
1804	return -2;
1805	}
1806	}
1807
1808	break;
1809	}
1810
1811	// reserve output buffer
1812	if (output_reserve(n_outputs_all) < n_outputs_all) {
1813	LLAMA_LOG_ERROR("%s: could not reserve space for batch with %d outputs\n", __func__, n_outputs_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %d outputs\n" , __func__, n_outputs_all);
1814	return -2;
1815	};
1816
1817	int64_t n_outputs_prev = 0;
1818	int64_t n_tokens_prev = 0;
1819
1820	do {
1821	const auto & ubatch = mctx->get_ubatch();
1822
1823	// count the outputs in this ubatch
1824	{
1825	int32_t n_outputs_new = 0;
1826
1827	if (n_outputs_all == n_tokens_all) {
1828	n_outputs_new = ubatch.n_tokens;
1829	} else {
1830	for (uint32_t i = 0; i < ubatch.n_tokens; i++) {
1831	n_outputs_new += (int32_t) (ubatch.output[i] != 0);
1832	}
1833	}
1834
1835	// needs to happen before the graph is built
1836	n_outputs = n_outputs_new;
1837	}
1838
1839	ggml_status status;
1840
1841	const auto * res = process_ubatch(ubatch, ctx_type_to_graph_type(cparams.ctx_type), mctx.get(), status);
1842
1843	if (!res) {
1844	// the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module
1845	llama_pos pos_min[LLAMA_MAX_SEQ256];
1846	for (int s = 0; s < LLAMA_MAX_SEQ256; ++s) {
1847	pos_min[s] = std::numeric_limits<llama_pos>::max();
1848	}
1849
1850	for (uint32_t i = 0; i < ubatch.n_tokens; ++i) {
1851	const auto & seq_id = ubatch.seq_id[i][0];
1852
1853	pos_min[seq_id] = std::min(pos_min[seq_id], ubatch.pos[i]);
1854	}
1855
1856	for (int s = 0; s < LLAMA_MAX_SEQ256; ++s) {
1857	if (pos_min[s] == std::numeric_limits<llama_pos>::max()) {
1858	continue;
1859	}
1860
1861	LLAMA_LOG_WARN("%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\n", __func__, s, pos_min[s])llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: removing memory module entries for seq_id = %d, pos = [%d, +inf)\n" , __func__, s, pos_min[s]);
1862
1863	memory->seq_rm(s, pos_min[s], -1);
1864	}
1865
1866	switch (status) {
1867	case GGML_STATUS_ABORTED: return 2;
1868	case GGML_STATUS_ALLOC_FAILED: return -2;
1869	case GGML_STATUS_FAILED: return -3;
1870	case GGML_STATUS_SUCCESS: GGML_ABORT("should not happen")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1870, "should not happen");
1871	}
1872	}
1873
1874	// plot the computation graph in dot format (for debugging purposes)
1875	//if (n_past%100 == 0) {
1876	// ggml_graph_dump_dot(gf, NULL, "llama.dot");
1877	//}
1878
1879	auto * t_logits = res->get_logits();
1880	auto * t_embd = cparams.embeddings ? res->get_embd() : nullptr;
1881	auto * t_h_nextn = cparams.embeddings_nextn ? res->get_h_nextn() : nullptr;
1882
1883	if (t_embd && res->get_embd_pooled()) {
1884	t_embd = res->get_embd_pooled();
1885	}
1886
1887	// extract logits
1888	if (logits.data && t_logits && n_outputs > 0 && needs_raw_logits(ubatch, sampling.samplers)) {
1889	ggml_backend_t backend_res = ggml_backend_sched_get_tensor_backend(sched.get(), t_logits);
1890	GGML_ASSERT(backend_res != nullptr)if (!(backend_res != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1890, "GGML_ASSERT(%s) failed", "backend_res != nullptr");
1891	GGML_ASSERT(logits.data != nullptr)if (!(logits.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1891, "GGML_ASSERT(%s) failed", "logits.data != nullptr");
1892
1893	float * logits_out = logits.data + n_outputs_prev*n_vocab;
1894
1895	if (n_outputs) {
1896	GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all)if (!(n_outputs_prev + n_outputs <= n_outputs_all)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1896, "GGML_ASSERT(%s) failed", "n_outputs_prev + n_outputs <= n_outputs_all" );
1897	GGML_ASSERT((n_outputs_prev + n_outputs)n_vocab <= (int64_t) logits.size)if (!((n_outputs_prev + n_outputs)n_vocab <= (int64_t) logits .size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1897, "GGML_ASSERT(%s) failed", "(n_outputs_prev + n_outputs)*n_vocab <= (int64_t) logits.size" );
1898	ggml_backend_tensor_get_async(backend_res, t_logits, logits_out, 0, n_outputsn_vocabsizeof(float));
1899	}
1900	}
1901
1902	// extract embeddings
1903	if (embd.data && t_embd && n_outputs > 0) {
1904	ggml_backend_t backend_embd = ggml_backend_sched_get_tensor_backend(sched.get(), t_embd);
1905	GGML_ASSERT(backend_embd != nullptr)if (!(backend_embd != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1905, "GGML_ASSERT(%s) failed", "backend_embd != nullptr");
1906
1907	switch (cparams.pooling_type) {
1908	case LLAMA_POOLING_TYPE_NONE:
1909	{
1910	// extract token embeddings
1911	GGML_ASSERT(embd.data != nullptr)if (!(embd.data != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1911, "GGML_ASSERT(%s) failed", "embd.data != nullptr");
1912	const uint32_t n_embd_out = hparams.n_embd_out();
1913	float * embd_out = embd.data + n_outputs_prev*n_embd_out;
1914
1915	if (n_outputs) {
1916	GGML_ASSERT( n_outputs_prev + n_outputs <= n_outputs_all)if (!(n_outputs_prev + n_outputs <= n_outputs_all)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1916, "GGML_ASSERT(%s) failed", "n_outputs_prev + n_outputs <= n_outputs_all" );
1917	GGML_ASSERT((n_outputs_prev + n_outputs)n_embd_out <= (int64_t) embd.size)if (!((n_outputs_prev + n_outputs)n_embd_out <= (int64_t) embd.size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1917, "GGML_ASSERT(%s) failed", "(n_outputs_prev + n_outputs)*n_embd_out <= (int64_t) embd.size" );
1918	ggml_backend_tensor_get_async(backend_embd, t_embd, embd_out, 0, n_outputsn_embd_outsizeof(float));
1919	}
1920	} break;
1921	case LLAMA_POOLING_TYPE_MEAN:
1922	case LLAMA_POOLING_TYPE_CLS:
1923	case LLAMA_POOLING_TYPE_LAST:
1924	{
1925	// extract sequence embeddings (cleared before processing each batch)
1926	auto & embd_seq_out = embd_seq;
1927
1928	// use n_embd_out (not n_embd_inp) - the pooled embedding has the model's
1929	// output dimension, which differs from input dimension for deepstack models (e.g. qwen3vl)
1930	const uint32_t n_embd_out = hparams.n_embd_out();
1931
1932	for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
1933	const llama_seq_id seq_id = ubatch.seq_id_unq[s];
1934	const int32_t seq_idx = ubatch.seq_idx[seq_id];
1935
1936	embd_seq_out[seq_id].resize(n_embd_out);
1937	ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_embd_outseq_idx)sizeof(float), n_embd_out*sizeof(float));
1938	}
1939	} break;
1940	case LLAMA_POOLING_TYPE_RANK:
1941	{
1942	// extract the rerank score - n_cls_out floats per sequence
1943	auto & embd_seq_out = embd_seq;
1944
1945	const uint32_t n_cls_out = hparams.n_cls_out;
1946
1947	for (uint32_t s = 0; s < ubatch.n_seqs_unq; ++s) {
1948	const llama_seq_id seq_id = ubatch.seq_id_unq[s];
1949	const int32_t seq_idx = ubatch.seq_idx[seq_id];
1950
1951	embd_seq_out[seq_id].resize(n_cls_out);
1952	ggml_backend_tensor_get_async(backend_embd, t_embd, embd_seq_out[seq_id].data(), (n_cls_outseq_idx)sizeof(float), n_cls_out*sizeof(float));
1953	}
1954	} break;
1955	case LLAMA_POOLING_TYPE_UNSPECIFIED:
1956	{
1957	GGML_ABORT("unknown pooling type")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1957, "unknown pooling type");
1958	}
1959	}
1960	}
1961
1962	extract_layer_inputs(res, n_tokens_prev, ubatch.n_tokens);
1963
1964	// extract nextn embeddings before
1965	// only meaningful in LLAMA_POOLING_TYPE_NONE (per-token); other pooling modes are ignored.
1966	{
1967	const bool masked = cparams.embeddings_nextn_masked;
1968	const int64_t n_rows = masked ? n_outputs : (int64_t) ubatch.n_tokens;
1969	const int64_t offset = masked ? n_outputs_prev : n_tokens_prev;
1970
1971	if (embd_nextn.data && t_h_nextn && n_rows > 0 && cparams.pooling_type == LLAMA_POOLING_TYPE_NONE) {
1972	ggml_backend_t backend_h = ggml_backend_sched_get_tensor_backend(sched.get(), t_h_nextn);
1973	GGML_ASSERT(backend_h != nullptr)if (!(backend_h != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1973, "GGML_ASSERT(%s) failed", "backend_h != nullptr");
1974
1975	const uint32_t n_embd = hparams.n_embd_out();
1976	float * embd_nextn_out = embd_nextn.data + offset*n_embd;
1977
1978	GGML_ASSERT((offset + n_rows)n_embd <= (int64_t) embd_nextn.size)if (!((offset + n_rows)n_embd <= (int64_t) embd_nextn.size )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 1978, "GGML_ASSERT(%s) failed", "(offset + n_rows)*n_embd <= (int64_t) embd_nextn.size" );
1979	ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn_out, 0, n_rowsn_embdsizeof(float));
1980	}
1981	}
1982
1983	// Copy backend sampling output if this ubatch produced any sampling tensors.
1984	if (has_samplers && (!res->t_sampled.empty() \|\| !res->t_sampled_probs.empty() \|\| !res->t_sampled_logits.empty())) {
1985	const auto seq_to_output_row = build_seq_to_output_row(ubatch, n_outputs_prev);
1986	const auto stride = n_vocab;
1987
1988	// async copy the sampling data from the backend to the host
1989	copy_tensor_async_ints(res->t_sampled, sampling.sampled, seq_to_output_row, sched.get());
1990
1991	copy_tensor_async_floats (res->t_sampled_logits, sampling.logits, stride, sampling.logits_count, seq_to_output_row, sched.get());
1992	copy_tensor_async_floats (res->t_sampled_probs, sampling.probs, stride, sampling.probs_count, seq_to_output_row, sched.get());
1993	copy_tensor_async_candidates(res->t_candidates, sampling.candidates, stride, sampling.candidates_count, seq_to_output_row, sched.get());
1994	}
1995
1996	n_outputs_prev += n_outputs;
1997	n_tokens_prev += ubatch.n_tokens;
1998	} while (mctx->next());
1999
2000	// set to total number of outputs in the batch, for use in llama_get_logits_ith
2001	n_outputs = n_outputs_all;
2002
2003	// set output mappings
2004	if (n_outputs > 0) {
2005	bool sorted_output = true;
2006
2007	auto & out_ids = balloc->get_out_ids();
2008
2009	GGML_ASSERT(out_ids.size() == (size_t) n_outputs)if (!(out_ids.size() == (size_t) n_outputs)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2009, "GGML_ASSERT(%s) failed", "out_ids.size() == (size_t) n_outputs" );
2010
2011	for (int64_t i = 0; i < n_outputs; ++i) {
2012	int64_t out_id = out_ids[i];
2013	output_ids[out_id] = i;
2014	if (out_id != i) {
2015	sorted_output = false;
2016	}
2017	}
2018
2019	// make the outputs have the same order they had in the user-provided batch
2020	// note: this is mostly relevant for recurrent models atm
2021	if (!sorted_output && n_outputs > 1) {
2022	GGML_ASSERT((size_t) n_outputs == out_ids.size())if (!((size_t) n_outputs == out_ids.size())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2022, "GGML_ASSERT(%s) failed", "(size_t) n_outputs == out_ids.size()" );
2023
2024	// TODO: is there something more efficient which also minimizes swaps?
2025	// selection sort, to minimize swaps (from https://en.wikipedia.org/wiki/Selection_sort)
2026	for (uint32_t i = 0; i < n_outputs - 1; ++i) {
2027	uint32_t j_min = i;
2028	for (uint32_t j = i + 1; j < n_outputs; ++j) {
2029	if (out_ids[j] < out_ids[j_min]) {
2030	j_min = j;
2031	}
2032	}
2033	if (j_min == i) {
2034	continue;
2035	}
2036	std::swap(out_ids[i], out_ids[j_min]);
2037
2038	// remember the swaps and apply them lazily upon logits/embeddings access
2039	output_swaps.push_back({ i, j_min });
2040	}
2041
2042	std::fill(output_ids.begin(), output_ids.end(), -1);
2043
2044	for (uint32_t i = 0; i < n_outputs; ++i) {
2045	output_ids[out_ids[i]] = i;
2046	}
2047	}
2048	}
2049
2050	// wait for the computation to finish (automatically done when obtaining the model output)
2051	//synchronize();
2052
2053	return 0;
2054	}
2055
2056	//
2057	// output
2058	//
2059
2060	uint32_t llama_context::output_reserve(int32_t n_outputs) {
2061	const auto & hparams = model.hparams;
2062	const auto & vocab = model.vocab;
2063
2064	const int64_t n_outputs_max = std::max<int64_t>(n_outputs, n_seq_max());
2065
2066	const auto n_batch = cparams.n_batch;
2067	const auto n_vocab = vocab.n_tokens();
2068	const auto n_embd = hparams.n_embd;
2069	const auto n_embd_out = hparams.n_embd_out();
2070
2071	bool has_logits = true;
2072	bool has_embd = cparams.embeddings;
2073	bool has_embd_nextn = cparams.embeddings_nextn;
2074
2075	// TODO: hacky enc-dec support
2076	if (model.arch == LLM_ARCH_T5) {
2077	has_logits = true;
2078	has_embd = true;
2079	}
2080
2081	size_t backend_float_count = 0;
2082	size_t backend_token_count = 0;
2083	size_t embd_layer_inp_float_count = 0;
2084
2085	logits.size = has_logits ? n_vocab*n_outputs_max : 0;
2086	embd.size = has_embd ? n_embd_out*n_outputs_max : 0;
2087	embd_nextn.size = has_embd_nextn ? n_embd_out*n_outputs_max : 0;
2088
2089	if (has_embd_nextn && !cparams.embeddings_nextn_masked) {
2090	// unmasked: nextn row exists for every token in the batch, not just
2091	// those flagged via batch.logits[i] -> size by token count instead.
2092	embd_nextn.size = (size_t) n_embd_out * n_batch;
2093	}
2094
2095	for (bool enabled : cparams.embeddings_layer_inp) {
2096	if (enabled) {
2097	embd_layer_inp_float_count += (size_t) n_embd * n_batch;
2098	}
2099	}
2100
2101	// Allocate backend sampling output buffers if there are backend samplers configured.
2102	const bool has_sampling = !sampling.samplers.empty();
2103	if (has_sampling) {
2104	backend_float_count = 2 * n_vocab * n_outputs_max; // logits + probs
2105	backend_token_count = (1 + n_vocab) * n_outputs_max; // sampled + candidates
2106	}
2107
2108	if (output_ids.empty()) {
2109	// init, never resized afterwards
2110	output_ids.resize(n_batch);
2111	}
2112
2113	const size_t prev_size = buf_output ? ggml_backend_buffer_get_size(buf_output.get()) : 0;
2114	const size_t new_size =
2115	(logits.size + embd.size + embd_nextn.size + embd_layer_inp_float_count + backend_float_count) * sizeof(float) +
2116	( backend_token_count) * sizeof(llama_token);
2117
2118	// alloc only when more than the current capacity is required
2119	// TODO: also consider shrinking the buffer
2120	if (!buf_output \|\| prev_size < new_size) {
2121	if (buf_output) {
2122	#ifndef NDEBUG
2123	// This doesn't happen often, but may be annoying in some cases (like the HellaSwag benchmark)
2124	LLAMA_LOG_DEBUG("%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n", __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reallocating output buffer from size %.02f MiB to %.02f MiB\n" , __func__, prev_size / 1024.0 / 1024.0, new_size / 1024.0 / 1024.0 );
2125	#endif
2126	synchronize();
2127
2128	// TODO: not needed?
2129	buf_output = nullptr;
2130	logits.data = nullptr;
2131	embd.data = nullptr;
2132	embd_nextn.data = nullptr;
2133	for (auto & layer_inp : embd_layer_inp) {
2134	layer_inp = {nullptr, 0};
2135	}
2136	}
2137
2138	auto * buft = ggml_backend_cpu_buffer_type();
2139	// try to use the host buffer of the device where the output tensor is allocated for faster transfer to system memory
2140	auto * output_dev = model.dev_output();
2141	auto * output_dev_host_buft = output_dev ? ggml_backend_dev_host_buffer_type(output_dev) : nullptr;
2142	if (output_dev_host_buft) {
2143	buft = output_dev_host_buft;
2144	}
2145	buf_output.reset(ggml_backend_buft_alloc_buffer(buft, new_size));
2146	if (buf_output == nullptr) {
2147	LLAMA_LOG_ERROR("%s: failed to allocate output buffer of size %.2f MiB\n", __func__, new_size / (1024.0 * 1024.0))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate output buffer of size %.2f MiB\n" , __func__, new_size / (1024.0 * 1024.0));
2148	return 0;
2149	}
2150	ggml_backend_buffer_clear(buf_output.get(), 0);
2151	}
2152
2153	float * output_base = (float *) ggml_backend_buffer_get_base(buf_output.get());
2154
2155	size_t offset = 0;
2156	uint8_t * base = (uint8_t *) output_base;
2157
2158	logits = has_logits ? buffer_view<float>{output_base, logits.size} : buffer_view<float>{nullptr, 0};
2159	offset += logits.size * sizeof(float);
2160
2161	embd = has_embd ? buffer_view<float>{(float *) (base + offset), embd.size} : buffer_view<float>{nullptr, 0};
2162	offset += embd.size * sizeof(float);
2163
2164	embd_nextn = has_embd_nextn ? buffer_view<float>{(float *) (base + offset), embd_nextn.size} : buffer_view<float>{nullptr, 0};
2165	offset += embd_nextn.size * sizeof(float);
2166
2167	for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) {
2168	if (cparams.embeddings_layer_inp[il]) {
2169	embd_layer_inp[il] = buffer_view<float>{(float ) (base + offset), (size_t) n_embd n_batch};
2170	offset += embd_layer_inp[il].size * sizeof(float);
2171	} else {
2172	embd_layer_inp[il] = buffer_view<float>{nullptr, 0};
2173	}
2174	}
2175
2176	if (has_sampling) {
2177	sampling.logits = {(float ) (base + offset), (size_t)(n_vocabn_outputs_max)};
2178	offset += sampling.logits.size * sizeof(float);
2179
2180	sampling.probs = {(float ) (base + offset), (size_t)(n_vocabn_outputs_max)};
2181	offset += sampling.probs.size * sizeof(float);
2182
2183	sampling.sampled = {(llama_token *) (base + offset), (size_t)n_outputs_max};
2184	offset += sampling.sampled.size * sizeof(llama_token);
2185
2186	sampling.candidates = {(llama_token ) (base + offset), (size_t)(n_vocabn_outputs_max)};
2187	offset += sampling.candidates.size * sizeof(llama_token);
	Value stored to 'offset' is never read
2188
2189	// The count vectors keep track of the actual number of logits/probs/candidates
2190	// copied from the backend for each output row.
2191
2192	sampling.logits_count.resize(n_outputs_max);
2193	sampling.probs_count.resize(n_outputs_max);
2194	sampling.candidates_count.resize(n_outputs_max);
2195
2196	std::fill(sampling.logits_count.begin(), sampling.logits_count.end(), 0);
2197	std::fill(sampling.probs_count.begin(), sampling.probs_count.end(), 0);
2198	std::fill(sampling.candidates_count.begin(), sampling.candidates_count.end(), 0);
2199
2200	std::fill_n(sampling.sampled.data, sampling.sampled.size, LLAMA_TOKEN_NULL-1);
2201	} else {
2202	sampling.logits = {nullptr, 0};
2203	sampling.probs = {nullptr, 0};
2204	sampling.sampled = {nullptr, 0};
2205	sampling.candidates = {nullptr, 0};
2206
2207	sampling.logits_count.clear();
2208	sampling.probs_count.clear();
2209	sampling.candidates_count.clear();
2210	}
2211
2212	// set all ids as invalid (negative)
2213	std::fill(output_ids.begin(), output_ids.end(), -1);
2214
2215	this->n_outputs = 0;
2216
2217	GGML_ASSERT(n_outputs_max <= cparams.n_outputs_max)if (!(n_outputs_max <= cparams.n_outputs_max)) ggml_abort( "/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2217, "GGML_ASSERT(%s) failed", "n_outputs_max <= cparams.n_outputs_max" );
2218
2219	return n_outputs_max;
2220	}
2221
2222	void llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) {
2223	for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) {
2224	if (!cparams.embeddings_layer_inp[il]) {
2225	continue;
2226	}
2227	if (!embd_layer_inp[il].has_data()) {
2228	GGML_ABORT("output layer input buffer not allocated")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2228, "output layer input buffer not allocated");
2229	}
2230	ggml_tensor * t = res->get_layer_inp((int) il);
2231	if (!t) {
2232	GGML_ABORT("layer input tensor not found")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2232, "layer input tensor not found");
2233	}
2234
2235	const size_t nbytes = ggml_nbytes(t);
2236	const size_t nfloats = nbytes / sizeof(float);
2237	GGML_ASSERT(n_tokens > 0)if (!(n_tokens > 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2237, "GGML_ASSERT(%s) failed", "n_tokens > 0");
2238	GGML_ASSERT(nfloats % n_tokens == 0)if (!(nfloats % n_tokens == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2238, "GGML_ASSERT(%s) failed", "nfloats % n_tokens == 0");
2239
2240	const size_t row_floats = nfloats / n_tokens;
2241	const size_t dst_offset = token_offset * row_floats;
2242	GGML_ASSERT(dst_offset + nfloats <= embd_layer_inp[il].size)if (!(dst_offset + nfloats <= embd_layer_inp[il].size)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2242, "GGML_ASSERT(%s) failed", "dst_offset + nfloats <= embd_layer_inp[il].size" );
2243
2244	ggml_backend_t backend = ggml_backend_sched_get_tensor_backend(sched.get(), t);
2245	GGML_ASSERT(backend != nullptr)if (!(backend != nullptr)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2245, "GGML_ASSERT(%s) failed", "backend != nullptr");
2246	ggml_backend_tensor_get_async(backend, t, embd_layer_inp[il].data + dst_offset, 0, nbytes);
2247	}
2248	}
2249
2250	void llama_context::output_reorder() {
2251	const uint64_t n_vocab = model.vocab.n_tokens();
2252	const uint64_t n_embd = model.hparams.n_embd;
2253
2254	for (size_t s = 0; s < output_swaps.size(); ++s) {
2255	const uint64_t i0 = output_swaps[s].i0;
2256	const uint64_t i1 = output_swaps[s].i1;
2257
2258	if (logits.size > 0) {
2259	for (uint64_t k = 0; k < n_vocab; k++) {
2260	std::swap(logits.data[i0n_vocab + k], logits.data[i1n_vocab + k]);
2261	}
2262	}
2263
2264	if (embd.size > 0) {
2265	for (uint64_t k = 0; k < n_embd; k++) {
2266	std::swap(embd.data[i0n_embd + k], embd.data[i1n_embd + k]);
2267	}
2268	}
2269
2270	if (embd_nextn.size > 0) {
2271	for (uint64_t k = 0; k < n_embd; k++) {
2272	std::swap(embd_nextn.data[i0n_embd + k], embd_nextn.data[i1n_embd + k]);
2273	}
2274	}
2275
2276	if (embd_layer_inp.size() > 0) {
2277	for (int lid = 0; lid < (int) embd_layer_inp.size(); ++lid) {
2278	if (embd_layer_inp[lid].size > 0) {
2279	for (uint64_t k = 0; k < n_embd; ++k) {
2280	std::swap(embd_layer_inp[lid].data[i0n_embd + k], embd_layer_inp[lid].data[i1n_embd + k]);
2281	}
2282	}
2283	}
2284	}
2285
2286	if (!sampling.samplers.empty()) {
2287	assert(sampling.logits.size > 0)(static_cast <bool> (sampling.logits.size > 0) ? void (0) : __assert_fail ("sampling.logits.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2288	assert(sampling.probs.size > 0)(static_cast <bool> (sampling.probs.size > 0) ? void (0) : __assert_fail ("sampling.probs.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2289	assert(sampling.candidates.size > 0)(static_cast <bool> (sampling.candidates.size > 0) ? void (0) : __assert_fail ("sampling.candidates.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ ));
2290	assert(sampling.sampled.size > 0)(static_cast <bool> (sampling.sampled.size > 0) ? void (0) : __assert_fail ("sampling.sampled.size > 0", __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__));
2291	assert(sampling.logits_count.size() > 0)(static_cast <bool> (sampling.logits_count.size() > 0 ) ? void (0) : __assert_fail ("sampling.logits_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ ));
2292	assert(sampling.probs_count.size() > 0)(static_cast <bool> (sampling.probs_count.size() > 0 ) ? void (0) : __assert_fail ("sampling.probs_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ ));
2293	assert(sampling.candidates_count.size() > 0)(static_cast <bool> (sampling.candidates_count.size() > 0) ? void (0) : __assert_fail ("sampling.candidates_count.size() > 0" , __builtin_FILE (), __builtin_LINE (), __extension__ __PRETTY_FUNCTION__ ));
2294
2295	for (uint64_t k = 0; k < n_vocab; ++k) {
2296	std::swap(sampling.logits.data[i0n_vocab + k], sampling.logits.data[i1n_vocab + k]);
2297	}
2298
2299	for (uint64_t k = 0; k < n_vocab; ++k) {
2300	std::swap(sampling.probs.data[i0n_vocab + k], sampling.probs.data[i1n_vocab + k]);
2301	}
2302
2303	for (uint64_t k = 0; k < n_vocab; ++k) {
2304	std::swap(sampling.candidates.data[i0n_vocab + k], sampling.candidates.data[i1n_vocab + k]);
2305	}
2306
2307	std::swap(sampling.sampled.data[i0], sampling.sampled.data[i1]);
2308	std::swap(sampling.logits_count[i0], sampling.logits_count[i1]);
2309	std::swap(sampling.probs_count[i0], sampling.probs_count[i1]);
2310	std::swap(sampling.candidates_count[i0], sampling.candidates_count[i1]);
2311	}
2312	}
2313
2314	output_swaps.clear();
2315	}
2316
2317	//
2318	// graph
2319	//
2320
2321	uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const {
2322	if (model.arch == LLM_ARCH_QWEN3NEXT \|\| model.arch == LLM_ARCH_KIMI_LINEAR \|\| model.arch == LLM_ARCH_QWEN35 \|\| model.arch == LLM_ARCH_QWEN35MOE) {
2323	return std::max<uint32_t>(n_tokens * 40, 32u * model.n_tensors());
2324	}
2325	uint32_t res = std::max<uint32_t>(1024u, 8u*model.n_tensors());
2326	for (const auto & lora : model.loras) {
2327	res += lora->get_n_nodes();
2328	}
2329	return res;
2330	}
2331
2332	llm_graph_result * llama_context::get_gf_res_reserve() const {
2333	return static_cast<llm_graph_result *>(gf_res_reserve.get());
2334	}
2335
2336	ggml_cgraph * llama_context::graph_reserve(
2337	uint32_t n_tokens, uint32_t n_seqs, uint32_t n_outputs, const llama_memory_context_i * mctx, bool split_only, size_t * sizes) {
2338	LLAMA_LOG_DEBUG("%s: reserving a graph for ubatch with n_tokens = %4u, n_seqs = %2u, n_outputs = %4u\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reserving a graph for ubatch with n_tokens = %4u, n_seqs = %2u, n_outputs = %4u\n" , __func__, n_tokens, n_seqs, n_outputs);
2339	GGML_ASSERT(n_outputs >= 1)if (!(n_outputs >= 1)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2339, "GGML_ASSERT(%s) failed", "n_outputs >= 1");
2340
2341	if (n_tokens % n_seqs != 0) {
2342	n_tokens = ((n_tokens + (n_seqs - 1)) / n_seqs) * n_seqs; // round to next multiple of n_seqs
2343	LLAMA_LOG_DEBUG("%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n", __func__, n_tokens, n_seqs, n_outputs)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: making n_tokens a multiple of n_seqs - n_tokens = %u, n_seqs = %u, n_outputs = %u\n" , __func__, n_tokens, n_seqs, n_outputs);
2344	}
2345
2346	ggml_backend_sched_reset(sched.get());
2347
2348	// when the scheduler is reset, we cannot reuse the old graph, so we reset the previous graph result to prevent that
2349	gf_res_prev->reset();
2350
2351	// store the n_outputs as it is, and restore it afterwards
2352	// TODO: not sure if needed, might simplify in the future by removing this
2353	const auto save_n_outputs = this->n_outputs;
2354
2355	this->n_outputs = n_outputs;
2356
2357	llama_batch_allocr balloc(model.hparams.n_pos_per_embd());
2358	llama_ubatch ubatch = balloc.ubatch_reserve(n_tokens/n_seqs, n_seqs);
2359
2360	// set one output token per sequence in order to activate all backend samplers
2361	std::vector<llama_seq_id> seq_ids(n_seqs);
2362	for (uint32_t i = 0; i < n_seqs; ++i) {
2363	seq_ids[i] = i;
2364	ubatch.n_seq_id[i] = 1;
2365	ubatch.seq_id[i] = &seq_ids[i];
2366	ubatch.output[i] = true;
2367	}
2368
2369	auto * res = gf_res_reserve.get();
2370
2371	const auto gparams = graph_params(res, ubatch, mctx, ctx_type_to_graph_type(cparams.ctx_type));
2372
2373	res->reset();
2374
2375	auto * gf = model.build_graph(gparams);
2376
2377	this->n_outputs = save_n_outputs;
2378
2379	// initialize scheduler with the specified graph
2380	if (split_only) {
2381	if (sizes) {
2382	ggml_backend_sched_reserve_size(sched.get(), gf, sizes);
2383	} else {
2384	ggml_backend_sched_split_graph(sched.get(), gf);
2385	}
2386	} else if (!ggml_backend_sched_reserve(sched.get(), gf)) {
2387	GGML_ASSERT(!sizes)if (!(!sizes)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2387, "GGML_ASSERT(%s) failed", "!sizes");
2388	LLAMA_LOG_ERROR("%s: failed to allocate compute buffers\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to allocate compute buffers\n" , __func__);
2389	return nullptr;
2390	}
2391
2392	return gf;
2393	}
2394
2395	llm_graph_params llama_context::graph_params(
2396	llm_graph_result * res,
2397	const llama_ubatch & ubatch,
2398	const llama_memory_context_i * mctx,
2399	llm_graph_type gtype) const {
2400	return {
2401	/.arch =/ model.arch,
2402	/.hparams =/ model.hparams,
2403	/.cparams =/ cparams,
2404	/.ubatch =/ ubatch,
2405	/.gtype =/ gtype,
2406	/.sched =/ sched.get(),
2407	/.backend_cpu =/ backend_cpu,
2408	/.cvec =/ cvec.get(),
2409	/.loras =/ loras.get(),
2410	/.mctx =/ mctx,
2411	/.cross =/ &cross,
2412	/.samplers =/ sampling.samplers,
2413	/.n_outputs =/ n_outputs,
2414	/.cb =/ graph_get_cb(),
2415	/.res =/ res,
2416	};
2417	}
2418
2419	ggml_status llama_context::graph_compute(
2420	ggml_cgraph * gf,
2421	bool batched) {
2422	int n_threads = batched ? cparams.n_threads_batch : cparams.n_threads;
2423	ggml_threadpool_t tp = batched ? threadpool_batch : threadpool;
2424
2425	if (backend_cpu != nullptr) {
2426	auto * reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend_cpu));
2427	auto * set_threadpool_fn = (decltype(ggml_backend_cpu_set_threadpool) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_set_threadpool");
2428	if (set_threadpool_fn) {
2429	set_threadpool_fn(backend_cpu, tp);
2430	}
2431	}
2432
2433	// set the number of threads for all the backends
2434	for (const auto & set_n_threads_fn : set_n_threads_fns) {
2435	set_n_threads_fn.second(set_n_threads_fn.first, n_threads);
2436	}
2437
2438	auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf);
2439	if (status != GGML_STATUS_SUCCESS) {
2440	LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: ggml_backend_sched_graph_compute_async failed with error %d\n" , __func__, status);
2441	}
2442
2443	// fprintf(stderr, "splits: %d\n", ggml_backend_sched_get_n_splits(sched));
2444
2445	return status;
2446	}
2447
2448	llm_graph_cb llama_context::graph_get_cb() const {
2449	return [&](const llama_ubatch & ubatch, ggml_tensor * cur, const char * name, int il) {
2450	if (il >= 0) {
2451	ggml_format_name(cur, "%s-%d", name, il);
2452	} else {
2453	ggml_set_name(cur, name);
2454	}
2455
2456	// norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends
2457	// FIXME: fix in ggml_backend_sched
2458	const bool full_offload = model.n_gpu_layers() > model.hparams.n_layer_all;
2459	if (ubatch.n_tokens < 32 \|\| full_offload) {
2460	if (il != -1 && strcmp(name, "norm") == 0) {
2461	const auto & dev_layer = model.dev_layer(il);
2462	for (const auto & backend : backends) {
2463	if (ggml_backend_get_device(backend.get()) == dev_layer) {
2464	if (ggml_backend_supports_op(backend.get(), cur)) {
2465	ggml_backend_sched_set_tensor_backend(sched.get(), cur, backend.get());
2466	}
2467	}
2468	}
2469	}
2470	}
2471	};
2472	}
2473
2474	//
2475	// state save/load
2476	//
2477
2478	class llama_io_write_dummy : public llama_io_write_i {
2479	public:
2480	llama_io_write_dummy(bool skip_tensors) : skip_tensors(skip_tensors) {}
2481
2482	void write(const void * /* src */, size_t size) override {
2483	size_written += size;
2484	}
2485
2486	void write_tensor(ggml_tensor * /* tensor /, size_t / offset */, size_t size) override {
2487	if (skip_tensors) {
2488	return;
2489	}
2490
2491	size_written += size;
2492	}
2493
2494	size_t n_bytes() override {
2495	return size_written;
2496	}
2497
2498	private:
2499	const bool skip_tensors;
2500
2501	size_t size_written = 0;
2502	};
2503
2504	class llama_io_write_host : public llama_io_write_i {
2505	public:
2506	llama_io_write_host(
2507	uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
2508
2509	~llama_io_write_host() {
2510	// TODO: add backend support to batch tensor_get? or some other way to speed this up
2511	for (const auto & winfo : winfos) {
2512	ggml_backend_tensor_get(winfo.tensor, winfo.ptr, winfo.offset, winfo.size);
2513	}
2514	}
2515
2516	void write(const void * src, size_t size) override {
2517	if (size > buf_size) {
2518	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2519	}
2520	memcpy(ptr, src, size);
2521	ptr += size;
2522	size_written += size;
2523	buf_size -= size;
2524	}
2525
2526	void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2527	if (size > buf_size) {
2528	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2529	}
2530
2531	// save the write for later during destruction
2532	winfos.push_back({tensor, ptr, size, offset});
2533
2534	ptr += size;
2535	size_written += size;
2536	buf_size -= size;
2537	}
2538
2539	size_t n_bytes() override {
2540	return size_written;
2541	}
2542
2543	private:
2544	uint8_t * ptr;
2545	size_t buf_size = 0;
2546	size_t size_written = 0;
2547
2548	struct write_info {
2549	ggml_tensor * tensor;
2550	uint8_t * ptr;
2551	size_t size;
2552	size_t offset;
2553	};
2554	std::vector<write_info> winfos;
2555	};
2556
2557	class llama_io_read_host : public llama_io_read_i {
2558	public:
2559	llama_io_read_host(const uint8_t * p, size_t len) : ptr(p), buf_size(len) {}
2560
2561	~llama_io_read_host() {
2562	// flush the reads
2563	for (const auto & rinfo : rinfos) {
2564	ggml_backend_tensor_set(rinfo.tensor, rinfo.ptr, rinfo.offset, rinfo.size);
2565	}
2566	}
2567
2568	void read(void * dst, size_t size) override {
2569	if (size > buf_size) {
2570	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2571	}
2572	memcpy(dst, ptr, size);
2573	ptr += size;
2574	size_read += size;
2575	buf_size -= size;
2576	}
2577
2578	void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2579	if (size > buf_size) {
2580	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2581	}
2582
2583	// save for later during destruction
2584	rinfos.push_back({tensor, ptr, size, offset});
2585
2586	ptr += size;
2587	size_read += size;
2588	buf_size -= size;
2589	}
2590
2591	size_t n_bytes() override {
2592	return size_read;
2593	}
2594
2595	private:
2596	const uint8_t * ptr;
2597	size_t buf_size = 0;
2598	size_t size_read = 0;
2599
2600	struct read_info {
2601	ggml_tensor * tensor;
2602	const uint8_t * ptr;
2603	size_t size;
2604	size_t offset;
2605	};
2606	std::vector<read_info> rinfos;
2607	};
2608
2609	class llama_io_write_file : public llama_io_write_i {
2610	public:
2611	llama_io_write_file(llama_file * f) : file(f) {}
2612
2613	void write(const void * src, size_t size) override {
2614	file->write_raw(src, size);
2615	size_written += size;
2616	}
2617
2618	void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2619	temp_buffer.resize(size);
2620	ggml_backend_tensor_get(tensor, temp_buffer.data(), offset, size);
2621	write(temp_buffer.data(), temp_buffer.size());
2622	}
2623
2624	size_t n_bytes() override {
2625	return size_written;
2626	}
2627
2628	private:
2629	llama_file * file;
2630	size_t size_written = 0;
2631	std::vector<uint8_t> temp_buffer;
2632	};
2633
2634	class llama_io_read_file : public llama_io_read_i {
2635	public:
2636	llama_io_read_file(llama_file * f) : file(f) {}
2637
2638	void read(void * dst, size_t size) override {
2639	file->read_raw(dst, size);
2640	size_read += size;
2641	}
2642
2643	void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2644	temp_buffer.resize(size);
2645	read(temp_buffer.data(), size);
2646	ggml_backend_tensor_set(tensor, temp_buffer.data(), offset, size);
2647	}
2648
2649	size_t n_bytes() override {
2650	return size_read;
2651	}
2652
2653	private:
2654	llama_file * file;
2655	size_t size_read = 0;
2656	std::vector<uint8_t> temp_buffer;
2657	};
2658
2659	class llama_io_write_device : public llama_io_write_i {
2660	public:
2661	llama_io_write_device(uint8_t * p, size_t len, llama_memory_buffers & mbufs) : ptr(p), buf_size(len), mbufs(mbufs) {
2662	}
2663
2664	~llama_io_write_device() {
2665	llama_memory_buffers mbufs_new;
2666
2667	for (const auto & winfo : winfos) {
2668	auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer);
2669
2670	mbufs_new[buft].n_tensors++;
2671	mbufs_new[buft].total_size += winfo.size;
2672	}
2673
2674	for (auto & [buft, mbuf] : mbufs_new) {
2675	ggml_init_params params = {
2676	/.mem_size =/ 2mbuf.n_tensorsggml_tensor_overhead(),
2677	/.mem_buffer =/ NULL__null,
2678	/.no_alloc =/ true,
2679	};
2680
2681	mbuf.ctx.reset(ggml_init(params));
2682
2683	mbuf.org.reserve(mbuf.n_tensors);
2684	mbuf.cpy.reserve(mbuf.n_tensors);
2685	}
2686
2687	for (const auto & winfo : winfos) {
2688	auto * buft = ggml_backend_buffer_get_type(winfo.tensor->buffer);
2689
2690	const int64_t n = winfo.size/ggml_element_size(winfo.tensor);
2691
2692	auto & mbuf = mbufs_new[buft];
2693
2694	mbuf.org.push_back(ggml_view_1d (mbuf.ctx.get(), winfo.tensor, n, winfo.offset));
2695	mbuf.cpy.push_back(ggml_new_tensor_1d(mbuf.ctx.get(), winfo.tensor->type, n));
2696	}
2697
2698	for (auto & [buft, mbuf] : mbufs_new) {
2699	auto & mbuf_cur = mbufs[buft];
2700
2701	bool need_alloc = false;
2702
2703	need_alloc = need_alloc \|\| (!mbuf_cur.buf);
2704	need_alloc = need_alloc \|\| (mbuf_cur.org.size() != mbuf.org.size());
2705	need_alloc = need_alloc \|\| (mbuf_cur.total_size != mbuf.total_size);
2706
2707	if (!need_alloc) {
2708	for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
2709	auto * org0 = mbuf_cur.org[i];
2710	auto * org1 = mbuf.org[i];
2711
2712	if (!ggml_are_same_shape(org0, org1)) {
2713	need_alloc = true;
2714	break;
2715	}
2716
2717	if (org0->view_src != org1->view_src \|\| org0->view_offs != org1->view_offs) {
2718	need_alloc = true;
2719	break;
2720	}
2721	}
2722	}
2723
2724	if (need_alloc) {
2725	if (!mbuf_cur.buf \|\| mbuf_cur.total_size != mbuf.total_size) {
2726	mbuf_cur = std::move(mbuf);
2727
2728	mbuf_cur.buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(mbuf_cur.ctx.get(), buft));
2729
2730	LLAMA_LOG_INFO("%s: allocated '%s' buffer %.3f MiB\n", __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0/1024.0)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: allocated '%s' buffer %.3f MiB\n" , __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0 /1024.0);
2731	} else {
2732	//LLAMA_LOG_INFO("%s: reallocating tensors in '%s' buffer %.3f MiB\n", __func__, ggml_backend_buft_name(buft), mbuf.total_size/1024.0/1024.0);
2733
2734	// save the old buffer and allocate the new tensors in it
2735	auto buf = std::move(mbuf_cur.buf);
2736
2737	mbuf_cur = std::move(mbuf);
2738
2739	ggml_tallocr talloc = ggml_tallocr_new(buf.get());
2740
2741	for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
2742	ggml_backend_view_init(mbuf_cur.org[i]);
2743	ggml_tallocr_alloc(&talloc, mbuf_cur.cpy[i]);
2744	}
2745
2746	mbuf_cur.buf = std::move(buf);
2747	}
2748	}
2749
2750	for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
2751	ggml_backend_tensor_copy(mbuf_cur.org[i], mbuf_cur.cpy[i]);
2752	}
2753	}
2754	}
2755
2756	void write(const void * src, size_t size) override {
2757	if (size > buf_size) {
2758	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2759	}
2760	memcpy(ptr, src, size);
2761	ptr += size;
2762	size_written += size;
2763	buf_size -= size;
2764	}
2765
2766	void write_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2767	// save the write for later during destruction
2768	winfos.push_back({tensor, ptr, size, offset});
2769	}
2770
2771	size_t n_bytes() override {
2772	return size_written;
2773	}
2774
2775	private:
2776	uint8_t * ptr;
2777	size_t buf_size = 0;
2778	size_t size_written = 0;
2779
2780	struct write_info {
2781	ggml_tensor * tensor;
2782	uint8_t * ptr;
2783	size_t size;
2784	size_t offset;
2785	};
2786	std::vector<write_info> winfos;
2787
2788	llama_memory_buffers & mbufs;
2789	};
2790
2791	class llama_io_read_device : public llama_io_read_i {
2792	public:
2793	llama_io_read_device(const uint8_t * p, size_t len, const llama_memory_buffers & mbufs) : ptr(p), buf_size(len), mbufs(mbufs) {
2794	}
2795
2796	~llama_io_read_device() {
2797	llama_memory_buffers mbufs_new;
2798
2799	for (const auto & rinfo : rinfos) {
2800	auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer);
2801
2802	mbufs_new[buft].n_tensors++;
2803	mbufs_new[buft].total_size += rinfo.size;
2804	}
2805
2806	for (auto & [buft, mbuf] : mbufs_new) {
2807	ggml_init_params params = {
2808	/.mem_size =/ mbuf.n_tensors*ggml_tensor_overhead(),
2809	/.mem_buffer =/ NULL__null,
2810	/.no_alloc =/ true,
2811	};
2812
2813	mbuf.ctx.reset(ggml_init(params));
2814
2815	mbuf.org.reserve(mbuf.n_tensors);
2816	}
2817
2818	for (const auto & rinfo : rinfos) {
2819	auto * buft = ggml_backend_buffer_get_type(rinfo.tensor->buffer);
2820
2821	const int64_t n = rinfo.size/ggml_element_size(rinfo.tensor);
2822
2823	auto & mbuf = mbufs_new[buft];
2824
2825	mbuf.org.push_back(ggml_view_1d(mbuf.ctx.get(), rinfo.tensor, n, rinfo.offset));
2826
2827	ggml_backend_view_init(mbuf.org.back());
2828	}
2829
2830	for (auto & [buft, mbuf] : mbufs_new) {
2831	const auto & mbuf_cur = mbufs.at(buft);
2832
2833	if (!mbuf_cur.buf \|\| mbuf_cur.n_tensors != mbuf.n_tensors \|\| mbuf_cur.total_size != mbuf.total_size) {
2834	GGML_ABORT("%s: memory buffer mismatch\n", __func__)ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2834, "%s: memory buffer mismatch\n", __func__);
2835	}
2836
2837	for (size_t i = 0; i < mbuf_cur.org.size(); ++i) {
2838	ggml_backend_tensor_copy(mbuf_cur.cpy[i], mbuf.org[i]);
2839	}
2840	}
2841
2842	GGML_ASSERT(buf_size == 0)if (!(buf_size == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2842, "GGML_ASSERT(%s) failed", "buf_size == 0");
2843	}
2844
2845	void read(void * dst, size_t size) override {
2846	if (size > buf_size) {
2847	throwabort_with_suppression(); if (false) std::runtime_error("unexpectedly reached end of buffer");
2848	}
2849	memcpy(dst, ptr, size);
2850	ptr += size;
2851	size_read += size;
2852	buf_size -= size;
2853	}
2854
2855	void read_tensor(ggml_tensor * tensor, size_t offset, size_t size) override {
2856	// save for later during destruction
2857	rinfos.push_back({tensor, ptr, size, offset});
2858	}
2859
2860	size_t n_bytes() override {
2861	return size_read;
2862	}
2863
2864	private:
2865	const uint8_t * ptr;
2866	size_t buf_size = 0;
2867	size_t size_read = 0;
2868
2869	struct read_info {
2870	ggml_tensor * tensor;
2871	const uint8_t * ptr;
2872	size_t size;
2873	size_t offset;
2874	};
2875	std::vector<read_info> rinfos;
2876
2877	const llama_memory_buffers & mbufs;
2878	};
2879
2880	size_t llama_context::state_get_size() {
2881	llama_io_write_dummy io(false);
2882	tryif (true) {
2883	return state_write_data(io);
2884	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2885	LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error getting state size: %s\n" , __func__, err.what());
2886	return 0;
2887	}
2888	}
2889
2890	size_t llama_context::state_get_data(uint8_t * dst, size_t size) {
2891	llama_io_write_host io(dst, size);
2892	tryif (true) {
2893	return state_write_data(io);
2894	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2895	LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving state: %s\n" , __func__, err.what());
2896	return 0;
2897	}
2898	}
2899
2900	size_t llama_context::state_set_data(const uint8_t * src, size_t size) {
2901	llama_io_read_host io(src, size);
2902	tryif (true) {
2903	return state_read_data(io);
2904	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2905	LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading state: %s\n" , __func__, err.what());
2906	return 0;
2907	}
2908	}
2909
2910	static constexpr uint32_t io_magic = 0xaf143cd8;
2911
2912	size_t llama_context::state_seq_get_size(llama_seq_id seq_id, llama_state_seq_flags flags) {
2913	llama_io_write_dummy io(flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2);
2914	tryif (true) {
2915	io.write(&io_magic, sizeof(io_magic));
2916	io.write(&seq_id, sizeof(seq_id));
2917
2918	return state_seq_write_data(io, seq_id, flags);
2919	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2920	LLAMA_LOG_ERROR("%s: error getting state size: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error getting state size: %s\n" , __func__, err.what());
2921	return 0;
2922	}
2923	}
2924
2925	size_t llama_context::state_seq_get_data(llama_seq_id seq_id, uint8_t * dst, size_t size, llama_state_seq_flags flags) {
2926	std::unique_ptr<llama_io_write_i> io;
2927	if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2) {
2928	io = std::make_unique<llama_io_write_device>(dst, size, mem_storage[seq_id]);
2929	} else {
2930	io = std::make_unique<llama_io_write_host>(dst, size);
2931	}
2932
2933	tryif (true) {
2934	io->write(&io_magic, sizeof(io_magic));
2935	io->write(&seq_id, sizeof(seq_id));
2936
2937	return state_seq_write_data(*io, seq_id, flags);
2938	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2939	LLAMA_LOG_ERROR("%s: error saving state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving state: %s\n" , __func__, err.what());
2940	return 0;
2941	}
2942	}
2943
2944	size_t llama_context::state_seq_set_data(llama_seq_id seq_id, const uint8_t * src, size_t size, llama_state_seq_flags flags) {
2945	std::unique_ptr<llama_io_read_i> io;
2946	if (flags & LLAMA_STATE_SEQ_FLAGS_ON_DEVICE2) {
2947	// create a temporary io to read the magic and the src seq_id
2948	io = std::make_unique<llama_io_read_host>(src, size);
2949
2950	uint32_t magic_read;
2951	io->read(&magic_read, sizeof(magic_read));
2952	if (io_magic != magic_read) {
2953	throwabort_with_suppression(); if (false) std::runtime_error("wrong sequence state magic");
2954	}
2955
2956	llama_seq_id seq_id_read;
2957	io->read(&seq_id_read, sizeof(seq_id_read));
2958
2959	GGML_ASSERT(mem_storage.find(seq_id_read) != mem_storage.end())if (!(mem_storage.find(seq_id_read) != mem_storage.end())) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 2959, "GGML_ASSERT(%s) failed", "mem_storage.find(seq_id_read) != mem_storage.end()" );
2960
2961	io = std::make_unique<llama_io_read_device>(src, size, mem_storage[seq_id_read]);
2962	} else {
2963	io = std::make_unique<llama_io_read_host>(src, size);
2964	}
2965
2966	tryif (true) {
2967	uint32_t magic_read;
2968	io->read(&magic_read, sizeof(magic_read));
2969	if (io_magic != magic_read) {
2970	throwabort_with_suppression(); if (false) std::runtime_error("wrong sequence state magic");
2971	}
2972
2973	llama_seq_id seq_id_read;
2974	io->read(&seq_id_read, sizeof(seq_id_read));
2975
2976	return state_seq_read_data(*io, seq_id, flags);
2977	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
2978	LLAMA_LOG_ERROR("%s: error loading state: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading state: %s\n" , __func__, err.what());
2979	return 0;
2980	}
2981	}
2982
2983	bool llama_context::state_load_file(const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
2984	llama_file file(filepath, "rb");
2985
2986	// sanity checks
2987	{
2988	const uint32_t magic = file.read_u32();
2989	const uint32_t version = file.read_u32();
2990
2991	if (magic != LLAMA_SESSION_MAGIC0x6767736eu \|\| version != LLAMA_SESSION_VERSION9) {
2992	LLAMA_LOG_ERROR("%s: unknown (magic, version) for session file: %08x, %08x\n", __func__, magic, version)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unknown (magic, version) for session file: %08x, %08x\n" , __func__, magic, version);
2993	return false;
2994	}
2995	}
2996
2997	// load the prompt
2998	{
2999	const uint32_t n_token_count = file.read_u32();
3000
3001	if (n_token_count > n_token_capacity) {
3002	LLAMA_LOG_ERROR("%s: token count in session file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: token count in session file exceeded capacity! %u > %zu\n" , __func__, n_token_count, n_token_capacity);
3003	return false;
3004	}
3005
3006	file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
3007	*n_token_count_out = n_token_count;
3008	}
3009
3010	// restore the context state
3011	{
3012	const size_t n_state_size_cur = file.size() - file.tell();
3013
3014	llama_io_read_file io( &file);
3015	const size_t n_read = state_read_data(io);
3016
3017	if (n_read != n_state_size_cur) {
3018	LLAMA_LOG_ERROR("%s: did not read all of the session file data! size %zu, got %zu\n", __func__, n_state_size_cur, n_read)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: did not read all of the session file data! size %zu, got %zu\n" , __func__, n_state_size_cur, n_read);
3019	return false;
3020	}
3021	}
3022
3023	return true;
3024	}
3025
3026	bool llama_context::state_save_file(const char * filepath, const llama_token * tokens, size_t n_token_count) {
3027	llama_file file(filepath, "wb");
3028
3029	file.write_u32(LLAMA_SESSION_MAGIC0x6767736eu);
3030	file.write_u32(LLAMA_SESSION_VERSION9);
3031
3032	// save the prompt
3033	file.write_u32((uint32_t) n_token_count);
3034	file.write_raw(tokens, sizeof(llama_token) * n_token_count);
3035
3036	// save the context state using stream saving
3037	llama_io_write_file io(&file);
3038	state_write_data(io);
3039
3040	return true;
3041	}
3042
3043	size_t llama_context::state_seq_load_file(llama_seq_id seq_id, const char * filepath, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
3044	llama_file file(filepath, "rb");
3045
3046	// version checks
3047	{
3048	const uint32_t magic = file.read_u32();
3049	const uint32_t version = file.read_u32();
3050
3051	if (magic != LLAMA_STATE_SEQ_MAGIC0x67677371u \|\| version != LLAMA_STATE_SEQ_VERSION2) {
3052	LLAMA_LOG_ERROR("%s: unknown (magic, version) for sequence state file: %08x, %08x\n", __func__, magic, version)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: unknown (magic, version) for sequence state file: %08x, %08x\n" , __func__, magic, version);
3053	return 0;
3054	}
3055	}
3056
3057	// load the prompt
3058	{
3059	const uint32_t n_token_count = file.read_u32();
3060
3061	if (n_token_count > n_token_capacity) {
3062	LLAMA_LOG_ERROR("%s: token count in sequence state file exceeded capacity! %u > %zu\n", __func__, n_token_count, n_token_capacity)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: token count in sequence state file exceeded capacity! %u > %zu\n" , __func__, n_token_count, n_token_capacity);
3063	return 0;
3064	}
3065
3066	file.read_raw(tokens_out, sizeof(llama_token) * n_token_count);
3067	*n_token_count_out = n_token_count;
3068	}
3069
3070	// restore the context state
3071	{
3072	const size_t state_size = file.size() - file.tell();
3073	llama_io_read_file io(&file);
3074	const size_t nread = state_seq_read_data(io, seq_id, 0);
3075	if (!nread) {
3076	LLAMA_LOG_ERROR("%s: failed to restore sequence state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to restore sequence state\n" , __func__);
3077	return 0;
3078	}
3079	GGML_ASSERT(nread <= state_size)if (!(nread <= state_size)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3079, "GGML_ASSERT(%s) failed", "nread <= state_size");
3080	GGML_ASSERT(nread + sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count_out == file.tell())if (!(nread + sizeof(uint32_t) 3 + sizeof(llama_token) * n_token_count_out == file.tell())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3080, "GGML_ASSERT(%s) failed", "nread + sizeof(uint32_t) 3 + sizeof(llama_token) * *n_token_count_out == file.tell()" );
3081	}
3082
3083	return file.tell();
3084	}
3085
3086	size_t llama_context::state_seq_save_file(llama_seq_id seq_id, const char * filepath, const llama_token * tokens, size_t n_token_count) {
3087	llama_file file(filepath, "wb");
3088
3089	file.write_u32(LLAMA_STATE_SEQ_MAGIC0x67677371u);
3090	file.write_u32(LLAMA_STATE_SEQ_VERSION2);
3091
3092	// save the prompt
3093	file.write_u32((uint32_t) n_token_count);
3094	file.write_raw(tokens, sizeof(llama_token) * n_token_count);
3095
3096	// save the context state using stream saving
3097	llama_io_write_file io(&file);
3098	state_seq_write_data(io, seq_id, 0);
3099
3100	const size_t res = file.tell();
3101	GGML_ASSERT(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes())if (!(res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes())) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3101, "GGML_ASSERT(%s) failed", "res == sizeof(uint32_t) * 3 + sizeof(llama_token) * n_token_count + io.n_bytes()" );
3102
3103	return res;
3104	}
3105
3106	size_t llama_context::state_write_data(llama_io_write_i & io) {
3107	LLAMA_LOG_DEBUG("%s: writing state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: writing state\n" , __func__);
3108
3109	// write model info
3110	{
3111	LLAMA_LOG_DEBUG("%s: - writing model info\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - writing model info\n" , __func__);
3112
3113	const std::string arch_str = llm_arch_name(model.arch);
3114	io.write_string(arch_str);
3115	// TODO: add more model-specific info which should prevent loading the session file if not identical
3116	}
3117
3118	if (memory != nullptr) {
3119	LLAMA_LOG_DEBUG("%s: - writing memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - writing memory module\n" , __func__);
3120	memory->state_write(io);
3121	}
3122
3123	return io.n_bytes();
3124	}
3125
3126	size_t llama_context::state_read_data(llama_io_read_i & io) {
3127	LLAMA_LOG_DEBUG("%s: reading state\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: reading state\n" , __func__);
3128
3129	// read model info
3130	{
3131	LLAMA_LOG_DEBUG("%s: - reading model info\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - reading model info\n" , __func__);
3132
3133	const std::string cur_arch_str = llm_arch_name(model.arch);
3134
3135	std::string arch_str;
3136	io.read_string(arch_str);
3137	if (cur_arch_str != arch_str) {
3138	throwabort_with_suppression(); if (false) std::runtime_error(format("wrong model arch: '%s' instead of '%s'", arch_str.c_str(), cur_arch_str.c_str()));
3139	}
3140	// TODO: add more info which needs to be identical but which is not verified otherwise
3141	}
3142
3143	if (memory) {
3144	LLAMA_LOG_DEBUG("%s: - reading memory module\n", __func__)llama_log_internal(GGML_LOG_LEVEL_DEBUG, "%s: - reading memory module\n" , __func__);
3145
3146	memory->state_read(io);
3147	}
3148
3149	return io.n_bytes();
3150	}
3151
3152	size_t llama_context::state_seq_write_data(llama_io_write_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
3153	GGML_UNUSED(seq_id)(void)(seq_id);
3154
3155	if (memory) {
3156	memory->state_write(io, seq_id, flags);
3157	}
3158
3159	return io.n_bytes();
3160	}
3161
3162	size_t llama_context::state_seq_read_data(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) {
3163	GGML_UNUSED(seq_id)(void)(seq_id);
3164
3165	if (memory) {
3166	memory->state_read(io, seq_id, flags);
3167	}
3168
3169	return io.n_bytes();
3170	}
3171
3172	//
3173	// perf
3174	//
3175
3176	llama_perf_context_data llama_context::perf_get_data() const {
3177	llama_perf_context_data data = {};
3178
3179	data.t_start_ms = 1e-3 * t_start_us;
3180	data.t_load_ms = 1e-3 * t_load_us;
3181	data.t_p_eval_ms = 1e-3 * t_p_eval_us;
3182	data.t_eval_ms = 1e-3 * t_eval_us;
3183	data.n_p_eval = std::max(1, n_p_eval);
3184	data.n_eval = std::max(1, n_eval);
3185	data.n_reused = std::max(0, n_reused);
3186
3187	return data;
3188	}
3189
3190	void llama_context::perf_reset() {
3191	t_start_us = ggml_time_us();
3192	t_eval_us = n_eval = 0;
3193	t_p_eval_us = n_p_eval = 0;
3194	n_reused = 0;
3195	}
3196
3197	llama_memory_breakdown llama_context::memory_breakdown() const {
3198	std::map<ggml_backend_buffer_type_t, llama_memory_breakdown_data> ret;
3199	for (const auto & [buft, size] : model.memory_breakdown()) {
3200	ret[buft].model += size;
3201	}
3202	if (memory) {
3203	for (const auto & [buft, size] : memory->memory_breakdown()) {
3204	ret[buft].context += size;
3205	}
3206	}
3207	if (model.hparams.no_alloc) {
3208	for (size_t i = 0; i < backends.size(); ++i) {
3209	ggml_backend_t backend = backends[i].get();
3210	ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
3211	ret[buft].compute += backend_buf_exp_size[i];
3212	}
3213	} else {
3214	for (const auto & backend_ptr : backends) {
3215	ggml_backend_t backend = backend_ptr.get();
3216	ggml_backend_buffer_type_t buft = ggml_backend_sched_get_buffer_type(sched.get(), backend);
3217	ret[buft].compute += ggml_backend_sched_get_buffer_size(sched.get(), backend);
3218	}
3219	}
3220	return ret;
3221	}
3222
3223	//
3224	// training
3225	//
3226
3227	static void llama_set_param(struct ggml_tensor * tensor, llama_opt_param_filter param_filter, void * userdata) {
3228	if (!tensor \|\| tensor->type != GGML_TYPE_F32) {
3229	return;
3230	}
3231	if (!param_filter(tensor, userdata)) {
3232	return;
3233	}
3234	if (strcmp(tensor->name, "token_embd.weight") == 0) {
3235	return; // FIXME
3236	}
3237	if (strcmp(tensor->name, "rope_freqs.weight") == 0) {
3238	return; // FIXME
3239	}
3240	ggml_set_param(tensor);
3241	}
3242
3243	void llama_context::opt_init(struct llama_model * model, struct llama_opt_params lopt_params) {
3244	GGML_ASSERT(!opt_ctx)if (!(!opt_ctx)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3244, "GGML_ASSERT(%s) failed", "!opt_ctx");
3245	model->hparams.n_ctx_train = lopt_params.n_ctx_train > 0 ? lopt_params.n_ctx_train : n_ctx();
3246	const uint32_t n_batch = std::min(this->n_batch(), model->hparams.n_ctx_train);
3247	const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
3248	GGML_ASSERT(model->hparams.n_ctx_train % n_batch == 0)if (!(model->hparams.n_ctx_train % n_batch == 0)) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3248, "GGML_ASSERT(%s) failed", "model->hparams.n_ctx_train % n_batch == 0" );
3249	GGML_ASSERT(n_batch % n_ubatch == 0)if (!(n_batch % n_ubatch == 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3249, "GGML_ASSERT(%s) failed", "n_batch % n_ubatch == 0");
3250
3251	ggml_opt_params opt_params = ggml_opt_default_params(sched.get(), GGML_OPT_LOSS_TYPE_CROSS_ENTROPY);
3252	opt_params.opt_period = n_batch / n_ubatch;
3253	opt_params.get_opt_pars = lopt_params.get_opt_pars;
3254	opt_params.get_opt_pars_ud = lopt_params.get_opt_pars_ud;
3255	opt_params.optimizer = lopt_params.optimizer_type;
3256	opt_ctx = ggml_opt_init(opt_params);
3257
3258	llama_opt_param_filter param_filter = lopt_params.param_filter;
3259	void * param_filter_ud = lopt_params.param_filter_ud;
3260
3261	//llama_set_param(model->tok_embd, param_filter, param_filter_ud); // FIXME
3262	llama_set_param(model->type_embd, param_filter, param_filter_ud);
3263	llama_set_param(model->pos_embd, param_filter, param_filter_ud);
3264	llama_set_param(model->tok_norm, param_filter, param_filter_ud);
3265	llama_set_param(model->tok_norm_b, param_filter, param_filter_ud);
3266	llama_set_param(model->output_norm, param_filter, param_filter_ud);
3267	llama_set_param(model->output_norm_b, param_filter, param_filter_ud);
3268	llama_set_param(model->output, param_filter, param_filter_ud);
3269	llama_set_param(model->output_b, param_filter, param_filter_ud);
3270	llama_set_param(model->output_norm_enc, param_filter, param_filter_ud);
3271	llama_set_param(model->cls, param_filter, param_filter_ud);
3272	llama_set_param(model->cls_b, param_filter, param_filter_ud);
3273	llama_set_param(model->cls_out, param_filter, param_filter_ud);
3274	llama_set_param(model->cls_out_b, param_filter, param_filter_ud);
3275	llama_set_param(model->cls_norm, param_filter, param_filter_ud);
3276
3277	for (struct llama_layer & layer : model->layers) {
3278	for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {
3279	llama_set_param(reinterpret_cast<struct ggml_tensor **>(&layer)[i], param_filter, param_filter_ud);
3280	}
3281	}
3282	}
3283
3284	void llama_context::opt_epoch_iter(
3285	ggml_opt_dataset_t dataset,
3286	ggml_opt_result_t result,
3287	const std::vector<llama_token> & tokens,
3288	const std::vector<llama_token> & labels_sparse,
3289	llama_batch & batch,
3290	ggml_opt_epoch_callback callback,
3291	bool train,
3292	int64_t idata_in_loop,
3293	int64_t ndata_in_loop,
3294	int64_t t_loop_start) {
3295	GGML_ASSERT(opt_ctx)if (!(opt_ctx)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3295, "GGML_ASSERT(%s) failed", "opt_ctx");
3296	const uint32_t n_ctx = llama_model_n_ctx_train(&model);
3297	const uint32_t n_batch = std::min(this->n_batch(), n_ctx);
3298	const uint32_t n_ubatch = std::min(this->n_ubatch(), n_batch);
3299
3300	memory->clear(true);
3301
3302	for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
3303	batch.n_tokens = n_batch;
3304	for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
3305	batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
3306	batch.pos [pos_batch] = pos_ctx + pos_batch;
3307	batch.n_seq_id[pos_batch] = 1;
3308	batch.seq_id [pos_batch][0] = 0;
3309	batch.logits [pos_batch] = true;
3310	}
3311
3312	if (!balloc->init(batch, model.vocab, nullptr, model.hparams.n_embd_inp(), cparams.kv_unified ? LLAMA_MAX_SEQ256 : cparams.n_seq_max, true)) {
3313	LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize batch\n" , __func__);
3314	return;
3315	}
3316
3317	const uint32_t n_tokens_all = balloc->get_n_tokens();
3318
3319	n_queued_tokens += n_tokens_all;
3320
3321	embd_seq.clear();
3322
3323	uint32_t n_outputs_all = n_tokens_all;
3324
3325	auto mctx = memory->init_batch(*balloc, cparams.n_ubatch, true);
3326	if (!mctx \|\| mctx->get_status() != LLAMA_MEMORY_STATUS_SUCCESS) {
3327	LLAMA_LOG_ERROR("%s: could not initialize batch\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not initialize batch\n" , __func__);
3328	break;
3329	}
3330
3331	// reserve output buffer
3332	if (output_reserve(n_outputs_all) < n_outputs_all) {
3333	LLAMA_LOG_ERROR("%s: could not reserve space for batch with %d outputs\n", __func__, n_outputs_all)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: could not reserve space for batch with %d outputs\n" , __func__, n_outputs_all);
3334	GGML_ABORT("TODO: handle this error")ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3334, "TODO: handle this error");
3335	};
3336
3337	uint32_t pos_batch = 0;
3338	do {
3339	const auto & ubatch = mctx->get_ubatch();
3340
3341	n_outputs = ubatch.n_tokens;
3342
3343	if (!mctx->apply()) {
3344	LLAMA_LOG_ERROR("%s: failed to update the memory context\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to update the memory context\n" , __func__);
3345	break;
3346	}
3347
3348	auto * res = gf_res_prev.get();
3349
3350	const auto gparams = graph_params(res, ubatch, mctx.get(), ctx_type_to_graph_type(cparams.ctx_type));
3351
3352	res->reset();
3353
3354	auto * gf = model.build_graph(gparams);
3355
3356	struct ggml_context * ctx_compute_opt;
3357	{
3358	const size_t size_gf = ggml_graph_size(gf);
3359	const size_t size_meta = 4size_gfggml_tensor_overhead() + 2ggml_graph_overhead_custom(size_gf, /grads = */ true);
3360	struct ggml_init_params params = {
3361	/.mem_size =/ size_meta,
3362	/.mem_buffer =/ nullptr,
3363	/.no_alloc =/ true,
3364	};
3365	ctx_compute_opt = ggml_init(params);
3366	}
3367	ggml_opt_prepare_alloc(opt_ctx, ctx_compute_opt, gf, res->get_inp_tokens(), res->get_logits());
3368	ggml_opt_alloc(opt_ctx, train);
3369
3370	res->set_inputs(&ubatch);
3371	{
3372	struct ggml_tensor * labels = ggml_opt_labels(opt_ctx);
3373	GGML_ASSERT(labels->ne[1] == n_ubatch)if (!(labels->ne[1] == n_ubatch)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3373, "GGML_ASSERT(%s) failed", "labels->ne[1] == n_ubatch" );
3374	ggml_set_zero(labels);
3375	const float onef = 1.0f;
3376	for (uint32_t pos_ubatch = 0; pos_ubatch < n_ubatch; ++pos_ubatch) {
3377	const uint32_t ilabel = pos_ctx + pos_batch + pos_ubatch;
3378	GGML_ASSERT(labels_sparse[ilabel] < labels->ne[0])if (!(labels_sparse[ilabel] < labels->ne[0])) ggml_abort ("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3378, "GGML_ASSERT(%s) failed", "labels_sparse[ilabel] < labels->ne[0]" );
3379	ggml_backend_tensor_set(labels, &onef, (pos_ubatchlabels->ne[0] + labels_sparse[ilabel])sizeof(float), sizeof(float));
3380	}
3381	}
3382	ggml_opt_eval(opt_ctx, result);
3383	if (callback) {
3384	callback(train, opt_ctx, dataset, result, idata_in_loop + (pos_ctx + pos_batch)/n_ubatch + 1, ndata_in_loop, t_loop_start);
3385	}
3386	ggml_free(ctx_compute_opt);
3387
3388	pos_batch += ubatch.n_tokens;
3389	} while (mctx->next());
3390	}
3391	}
3392
3393	void llama_context::opt_epoch(
3394	ggml_opt_dataset_t dataset,
3395	ggml_opt_result_t result_train,
3396	ggml_opt_result_t result_eval,
3397	int64_t idata_split,
3398	ggml_opt_epoch_callback callback_train,
3399	ggml_opt_epoch_callback callback_eval) {
3400	const uint32_t n_ctx = this->n_ctx();
3401	const uint32_t n_batch = std::min(cparams.n_batch, n_ctx);
3402	const uint32_t n_ubatch = std::min(cparams.n_ubatch, n_batch);
3403	const int64_t ndata = ggml_opt_dataset_ndata(dataset);
3404
3405	GGML_ASSERT(idata_split >= 0)if (!(idata_split >= 0)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3405, "GGML_ASSERT(%s) failed", "idata_split >= 0");
3406	GGML_ASSERT(idata_split <= ndata)if (!(idata_split <= ndata)) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3406, "GGML_ASSERT(%s) failed", "idata_split <= ndata");
3407
3408	const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
3409
3410	struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
3411	std::vector<llama_token> tokens(n_ctx);
3412	std::vector<llama_token> labels_sparse(n_ctx);
3413
3414	int64_t idata = 0;
3415
3416	int64_t t_loop_start = ggml_time_us();
3417	int64_t ndata_in_loop = idata_split*ubatch_per_ctx;
3418	for (; idata < idata_split; ++idata) {
3419	constexpr bool train = true;
3420	const int64_t idata_in_loop = idata*ubatch_per_ctx;
3421
3422	ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
3423	opt_epoch_iter(dataset, result_train, tokens, labels_sparse, batch,
3424	callback_train, train, idata_in_loop, ndata_in_loop, t_loop_start);
3425	}
3426
3427	t_loop_start = ggml_time_us();
3428	ndata_in_loop = (ndata - idata_split)*ubatch_per_ctx;
3429	for (; idata < ndata; ++idata) {
3430	constexpr bool train = false;
3431	const int64_t idata_in_loop = (idata - idata_split)*ubatch_per_ctx;
3432
3433	ggml_opt_dataset_get_batch_host(dataset, tokens.data(), n_ctx*sizeof(llama_token), labels_sparse.data(), idata);
3434	opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
3435	callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
3436	}
3437
3438	llama_batch_free(batch);
3439	}
3440
3441	//
3442	// interface implementation
3443	//
3444
3445	llama_context_params llama_context_default_params() {
3446	llama_context_params result = {
3447	/.n_ctx =/ 512,
3448	/.n_batch =/ 2048,
3449	/.n_ubatch =/ 512,
3450	/.n_seq_max =/ 1,
3451	/.n_rs_seq =/ 0,
3452	/.n_outputs_max =/ 0,
3453	/.n_threads =/ GGML_DEFAULT_N_THREADS4, // TODO: better default
3454	/.n_threads_batch =/ GGML_DEFAULT_N_THREADS4,
3455	/.ctx_type =/ LLAMA_CONTEXT_TYPE_DEFAULT,
3456	/.rope_scaling_type =/ LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
3457	/.pooling_type =/ LLAMA_POOLING_TYPE_UNSPECIFIED,
3458	/.attention_type =/ LLAMA_ATTENTION_TYPE_UNSPECIFIED,
3459	/.flash_attn_type =/ LLAMA_FLASH_ATTN_TYPE_AUTO,
3460	/.rope_freq_base =/ 0.0f,
3461	/.rope_freq_scale =/ 0.0f,
3462	/.yarn_ext_factor =/ -1.0f,
3463	/.yarn_attn_factor =/ -1.0f,
3464	/.yarn_beta_fast =/ -1.0f,
3465	/.yarn_beta_slow =/ -1.0f,
3466	/.yarn_orig_ctx =/ 0,
3467	/.defrag_thold =/ -1.0f,
3468	/.cb_eval =/ nullptr,
3469	/.cb_eval_user_data =/ nullptr,
3470	/.type_k =/ GGML_TYPE_F16,
3471	/.type_v =/ GGML_TYPE_F16,
3472	/.abort_callback =/ nullptr,
3473	/.abort_callback_data =/ nullptr,
3474	/.embeddings =/ false,
3475	/.offload_kqv =/ true,
3476	/.no_perf =/ true,
3477	/.op_offload =/ true,
3478	/.swa_full =/ true,
3479	/.kv_unified =/ false,
3480	/.sampler =/ nullptr,
3481	/.n_sampler =/ 0,
3482	/.ctx_other =/ nullptr,
3483	};
3484
3485	return result;
3486	}
3487
3488	llama_context * llama_init_from_model(
3489	llama_model * model,
3490	llama_context_params params) {
3491	if (!model) {
3492	LLAMA_LOG_ERROR("%s: model cannot be NULL\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: model cannot be NULL\n" , __func__);
3493	return nullptr;
3494	}
3495
3496	if (params.n_batch == 0 && params.n_ubatch == 0) {
3497	LLAMA_LOG_ERROR("%s: n_batch and n_ubatch cannot both be zero\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_batch and n_ubatch cannot both be zero\n" , __func__);
3498	return nullptr;
3499	}
3500
3501	if (params.n_ctx == 0 && model->hparams.n_ctx_train == 0) {
3502	LLAMA_LOG_ERROR("%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: n_ctx and model->hparams.n_ctx_train cannot both be zero\n" , __func__);
3503	return nullptr;
3504	}
3505
3506	if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && model->arch == LLM_ARCH_GROK) {
3507	LLAMA_LOG_WARN("%s: flash_attn is not compatible with Grok - forcing off\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: flash_attn is not compatible with Grok - forcing off\n" , __func__);
3508	params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_DISABLED;
3509	}
3510
3511	if (model->split_mode() == LLAMA_SPLIT_MODE_TENSOR) {
3512	if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) {
3513	LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for SPLIT_MODE_TENSOR\n", __func__)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: enabling flash_attn since it is required for SPLIT_MODE_TENSOR\n" , __func__);
3514	params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
3515	}
3516	if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) {
3517	LLAMA_LOG_ERROR("%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: SPLIT_MODE_TENSOR requires flash_attn to be enabled\n" , __func__);
3518	return nullptr;
3519	}
3520	}
3521
3522	if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {
3523	const uint32_t blck_size = ggml_blck_size(params.type_k);
3524	for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {
3525	if (model->hparams.n_embd_head_k(il) % blck_size != 0) {
3526	LLAMA_LOG_ERROR("%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n" , __func__, ggml_type_name(params.type_k), blck_size, model-> hparams.n_embd_head_k(il))
3527	__func__, ggml_type_name(params.type_k), blck_size, model->hparams.n_embd_head_k(il))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n" , __func__, ggml_type_name(params.type_k), blck_size, model-> hparams.n_embd_head_k(il));
3528	return nullptr;
3529	}
3530	}
3531	}
3532
3533	if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_v)) {
3534	const uint32_t blck_size = ggml_blck_size(params.type_v);
3535	for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) {
3536	if (model->hparams.n_embd_head_v(il) % blck_size != 0) {
3537	LLAMA_LOG_ERROR("%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n",llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n" , __func__, ggml_type_name(params.type_v), blck_size, model-> hparams.n_embd_head_v(il))
3538	__func__, ggml_type_name(params.type_v), blck_size, model->hparams.n_embd_head_v(il))llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n" , __func__, ggml_type_name(params.type_v), blck_size, model-> hparams.n_embd_head_v(il));
3539	return nullptr;
3540	}
3541	}
3542	}
3543
3544	if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
3545	LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: V cache quantization requires flash_attn\n" , __func__);
3546	return nullptr;
3547	}
3548
3549	if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED &&
3550	params.pooling_type != model->hparams.pooling_type) {
3551	//user-specified pooling-type is different from the model default
3552	LLAMA_LOG_WARN("%s: model default pooling_type is [%d], but [%d] was specified\n", __func__,llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: model default pooling_type is [%d], but [%d] was specified\n" , __func__, model->hparams.pooling_type, params.pooling_type )
3553	model->hparams.pooling_type, params.pooling_type)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: model default pooling_type is [%d], but [%d] was specified\n" , __func__, model->hparams.pooling_type, params.pooling_type );
3554	}
3555
3556	if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP &&
3557	model->hparams.n_layer_nextn == 0) {
3558	LLAMA_LOG_WARN("%s: context type MTP requested but model doesn't contain MTP layers\n", __func__)llama_log_internal(GGML_LOG_LEVEL_WARN , "%s: context type MTP requested but model doesn't contain MTP layers\n" , __func__);
3559	return nullptr;
3560	}
3561
3562	tryif (true) {
3563	auto * ctx = new llama_context(*model, params);
3564	return ctx;
3565	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
3566	LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to initialize the context: %s\n" , __func__, err.what());
3567	}
3568
3569	return nullptr;
3570	}
3571
3572	// deprecated
3573	llama_context * llama_new_context_with_model(
3574	llama_model * model,
3575	llama_context_params params) {
3576	return llama_init_from_model(model, params);
3577	}
3578
3579	void llama_free(llama_context * ctx) {
3580	delete ctx;
3581	}
3582
3583	uint32_t llama_n_ctx(const llama_context * ctx) {
3584	return ctx->n_ctx();
3585	}
3586
3587	uint32_t llama_n_ctx_seq(const llama_context * ctx) {
3588	return ctx->n_ctx_seq();
3589	}
3590
3591	uint32_t llama_n_batch(const llama_context * ctx) {
3592	return ctx->n_batch();
3593	}
3594
3595	uint32_t llama_n_ubatch(const llama_context * ctx) {
3596	return ctx->n_ubatch();
3597	}
3598
3599	uint32_t llama_n_seq_max(const llama_context * ctx) {
3600	return ctx->n_seq_max();
3601	}
3602
3603	uint32_t llama_n_rs_seq(const llama_context * ctx) {
3604	return ctx->get_cparams().n_rs_seq;
3605	}
3606
3607	const llama_model * llama_get_model(const llama_context * ctx) {
3608	return &ctx->get_model();
3609	}
3610
3611	enum llama_pooling_type llama_pooling_type(const llama_context * ctx) {
3612	return ctx->pooling_type();
3613	}
3614
3615	void llama_attach_threadpool(
3616	llama_context * ctx,
3617	ggml_threadpool_t threadpool,
3618	ggml_threadpool_t threadpool_batch) {
3619	ctx->attach_threadpool(threadpool, threadpool_batch);
3620	}
3621
3622	void llama_detach_threadpool(llama_context * ctx) {
3623	ctx->detach_threadpool();
3624	}
3625
3626	void llama_set_n_threads(llama_context * ctx, int32_t n_threads, int32_t n_threads_batch) {
3627	ctx->set_n_threads(n_threads, n_threads_batch);
3628	}
3629
3630	int32_t llama_n_threads(llama_context * ctx) {
3631	return ctx->n_threads();
3632	}
3633
3634	int32_t llama_n_threads_batch(llama_context * ctx) {
3635	return ctx->n_threads_batch();
3636	}
3637
3638	void llama_set_abort_callback(llama_context * ctx, bool (abort_callback)(void data), void * abort_callback_data) {
3639	ctx->set_abort_callback(abort_callback, abort_callback_data);
3640	}
3641
3642	void llama_set_embeddings(llama_context * ctx, bool embeddings) {
3643	ctx->set_embeddings(embeddings);
3644	}
3645
3646	void llama_set_causal_attn(llama_context * ctx, bool causal_attn) {
3647	ctx->set_causal_attn(causal_attn);
3648	}
3649
3650	void llama_set_warmup(llama_context * ctx, bool warmup) {
3651	ctx->set_warmup(warmup);
3652	}
3653
3654	void llama_synchronize(llama_context * ctx) {
3655	ctx->synchronize();
3656	}
3657
3658	float * llama_get_logits(llama_context * ctx) {
3659	ctx->synchronize();
3660
3661	return ctx->get_logits();
3662	}
3663
3664	float * llama_get_logits_ith(llama_context * ctx, int32_t i) {
3665	ctx->synchronize();
3666
3667	float * res = nullptr;
3668
3669	res = ctx->get_sampled_logits_ith(i);
3670
3671	if (!res) {
3672	res = ctx->get_logits_ith(i);
3673	}
3674
3675	return res;
3676	}
3677
3678	float * llama_get_embeddings(llama_context * ctx) {
3679	ctx->synchronize();
3680
3681	return ctx->get_embeddings();
3682	}
3683
3684	float * llama_get_embeddings_ith(llama_context * ctx, int32_t i) {
3685	ctx->synchronize();
3686
3687	return ctx->get_embeddings_ith(i);
3688	}
3689
3690	float * llama_get_embeddings_seq(llama_context * ctx, llama_seq_id seq_id) {
3691	ctx->synchronize();
3692
3693	return ctx->get_embeddings_seq(seq_id);
3694	}
3695
3696	void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) {
3697	ctx->set_embeddings_nextn(value, masked);
3698	}
3699
3700	void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) {
3701	ctx->set_embeddings_layer_inp(lid, value);
3702	}
3703
3704	llama_memory_t llama_get_memory(const struct llama_context * ctx) {
3705	if (!ctx) {
3706	return nullptr;
3707	}
3708
3709	return ctx->get_memory();
3710	}
3711
3712	float * llama_get_embeddings_nextn(llama_context * ctx) {
3713	ctx->synchronize();
3714
3715	return ctx->get_embeddings_nextn();
3716	}
3717
3718	float * llama_get_embeddings_nextn_ith(llama_context * ctx, int32_t i) {
3719	ctx->synchronize();
3720
3721	return ctx->get_embeddings_nextn_ith(i);
3722	}
3723
3724	float * llama_get_embeddings_layer_inp(llama_context * ctx, uint32_t lid) {
3725	ctx->synchronize();
3726
3727	return ctx->get_embeddings_layer_inp(lid);
3728	}
3729
3730	bool llama_set_sampler(llama_context * ctx, llama_seq_id seq_id, llama_sampler * smpl) {
3731	return ctx->set_sampler(seq_id, smpl);
3732	}
3733
3734	llama_token llama_get_sampled_token_ith(llama_context * ctx, int32_t i) {
3735	ctx->synchronize();
3736
3737	return ctx->get_sampled_token_ith(i);
3738	}
3739
3740	float * llama_get_sampled_probs_ith(llama_context * ctx, int32_t i) {
3741	ctx->synchronize();
3742
3743	return ctx->get_sampled_probs_ith(i);
3744	}
3745
3746	float * llama_get_sampled_logits_ith(llama_context * ctx, int32_t i) {
3747	ctx->synchronize();
3748
3749	return ctx->get_sampled_logits_ith(i);
3750	}
3751
3752	llama_token * llama_get_sampled_candidates_ith(llama_context * ctx, int32_t i) {
3753	ctx->synchronize();
3754
3755	return const_cast<llama_token *>(ctx->get_sampled_candidates_ith(i));
3756	}
3757
3758	uint32_t llama_get_sampled_candidates_count_ith(llama_context * ctx, int32_t i) {
3759	ctx->synchronize();
3760
3761	return static_cast<uint32_t>(ctx->get_sampled_candidates_count(i));
3762	}
3763
3764	uint32_t llama_get_sampled_logits_count_ith(llama_context * ctx, int32_t i) {
3765	ctx->synchronize();
3766
3767	return static_cast<uint32_t>(ctx->get_sampled_logits_count(i));
3768	}
3769
3770	uint32_t llama_get_sampled_probs_count_ith(llama_context * ctx, int32_t i) {
3771	ctx->synchronize();
3772
3773	return static_cast<uint32_t>(ctx->get_sampled_probs_count(i));
3774	}
3775
3776	struct ggml_cgraph * llama_graph_reserve(
3777	struct llama_context * ctx,
3778	uint32_t n_tokens,
3779	uint32_t n_seqs,
3780	uint32_t n_outputs) {
3781	auto memory = ctx->get_memory();
3782	llama_memory_context_ptr mctx;
3783	if (memory) {
3784	mctx = memory->init_full();
3785	}
3786	return ctx->graph_reserve(n_tokens, n_seqs, n_outputs, mctx.get());
3787	}
3788
3789	// llama adapter API
3790
3791	int32_t llama_set_adapters_lora(
3792	llama_context * ctx,
3793	llama_adapter_lora ** adapters,
3794	size_t n_adapters,
3795	float * scales) {
3796	if (adapters == nullptr \|\| scales == nullptr) {
3797	GGML_ASSERT(n_adapters == 0 && "invalid llama_set_adapters_lora call")if (!(n_adapters == 0 && "invalid llama_set_adapters_lora call" )) ggml_abort("/root/firefox-clang/third_party/llama.cpp/src/llama-context.cpp" , 3797, "GGML_ASSERT(%s) failed", "n_adapters == 0 && \"invalid llama_set_adapters_lora call\"" );
3798	}
3799
3800	ctx->set_adapters_lora(adapters, n_adapters, scales);
3801
3802	return 0;
3803	}
3804
3805	int32_t llama_set_adapter_cvec(
3806	llama_context * ctx,
3807	const float * data,
3808	size_t len,
3809	int32_t n_embd,
3810	int32_t il_start,
3811	int32_t il_end) {
3812	bool res = ctx->set_adapter_cvec(data, len, n_embd, il_start, il_end);
3813
3814	return res ? 0 : -1;
3815	}
3816
3817	//
3818	// memory
3819	//
3820
3821	void llama_memory_clear(llama_memory_t mem, bool data) {
3822	if (!mem) {
3823	return;
3824	}
3825
3826	mem->clear(data);
3827	}
3828
3829	bool llama_memory_seq_rm(
3830	llama_memory_t mem,
3831	llama_seq_id seq_id,
3832	llama_pos p0,
3833	llama_pos p1) {
3834	if (!mem) {
3835	return true;
3836	}
3837
3838	return mem->seq_rm(seq_id, p0, p1);
3839	}
3840
3841	void llama_memory_seq_cp(
3842	llama_memory_t mem,
3843	llama_seq_id seq_id_src,
3844	llama_seq_id seq_id_dst,
3845	llama_pos p0,
3846	llama_pos p1) {
3847	if (!mem) {
3848	return;
3849	}
3850
3851	mem->seq_cp(seq_id_src, seq_id_dst, p0, p1);
3852	}
3853
3854	void llama_memory_seq_keep(
3855	llama_memory_t mem,
3856	llama_seq_id seq_id) {
3857	if (!mem) {
3858	return;
3859	}
3860
3861	mem->seq_keep(seq_id);
3862	}
3863
3864	void llama_memory_seq_add(
3865	llama_memory_t mem,
3866	llama_seq_id seq_id,
3867	llama_pos p0,
3868	llama_pos p1,
3869	llama_pos delta) {
3870	if (!mem) {
3871	return;
3872	}
3873
3874	mem->seq_add(seq_id, p0, p1, delta);
3875	}
3876
3877	void llama_memory_seq_div(
3878	llama_memory_t mem,
3879	llama_seq_id seq_id,
3880	llama_pos p0,
3881	llama_pos p1,
3882	int d) {
3883	if (!mem) {
3884	return;
3885	}
3886
3887	mem->seq_div(seq_id, p0, p1, d);
3888	}
3889
3890	llama_pos llama_memory_seq_pos_min(
3891	llama_memory_t mem,
3892	llama_seq_id seq_id) {
3893	if (!mem) {
3894	return -1;
3895	}
3896
3897	return mem->seq_pos_min(seq_id);
3898	}
3899
3900	llama_pos llama_memory_seq_pos_max(
3901	llama_memory_t mem,
3902	llama_seq_id seq_id) {
3903	if (!mem) {
3904	return -1;
3905	}
3906
3907	return mem->seq_pos_max(seq_id);
3908	}
3909
3910	bool llama_memory_can_shift(llama_memory_t mem) {
3911	if (!mem) {
3912	return false;
3913	}
3914
3915	return mem->get_can_shift();
3916	}
3917
3918	// llama state API
3919
3920	// deprecated
3921	size_t llama_get_state_size(llama_context * ctx) {
3922	return llama_state_get_size(ctx);
3923	}
3924
3925	// deprecated
3926	size_t llama_copy_state_data(llama_context * ctx, uint8_t * dst) {
3927	return llama_state_get_data(ctx, dst, -1);
3928	}
3929
3930	// deprecated
3931	size_t llama_set_state_data(llama_context * ctx, const uint8_t * src) {
3932	return llama_state_set_data(ctx, src, -1);
3933	}
3934
3935	// deprecated
3936	bool llama_load_session_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
3937	return llama_state_load_file(ctx, path_session, tokens_out, n_token_capacity, n_token_count_out);
3938	}
3939
3940	// deprecated
3941	bool llama_save_session_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
3942	return llama_state_save_file(ctx, path_session, tokens, n_token_count);
3943	}
3944
3945	// Returns the actual size of the state.
3946	// Intended to be used when saving to state to a buffer.
3947	size_t llama_state_get_size(llama_context * ctx) {
3948	return ctx->state_get_size();
3949	}
3950
3951	size_t llama_state_get_data(llama_context * ctx, uint8_t * dst, size_t size) {
3952	ctx->synchronize();
3953
3954	return ctx->state_get_data(dst, size);
3955	}
3956
3957	// Sets the state reading from the specified source address
3958	size_t llama_state_set_data(llama_context * ctx, const uint8_t * src, size_t size) {
3959	ctx->synchronize();
3960
3961	return ctx->state_set_data(src, size);
3962	}
3963
3964	bool llama_state_load_file(llama_context * ctx, const char * path_session, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
3965	ctx->synchronize();
3966
3967	tryif (true) {
3968	return ctx->state_load_file(path_session, tokens_out, n_token_capacity, n_token_count_out);
3969	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
3970	LLAMA_LOG_ERROR("%s: error loading session file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading session file: %s\n" , __func__, err.what());
3971	return false;
3972	}
3973	}
3974
3975	bool llama_state_save_file(llama_context * ctx, const char * path_session, const llama_token * tokens, size_t n_token_count) {
3976	ctx->synchronize();
3977
3978	tryif (true) {
3979	return ctx->state_save_file(path_session, tokens, n_token_count);
3980	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
3981	LLAMA_LOG_ERROR("%s: error saving session file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving session file: %s\n" , __func__, err.what());
3982	return false;
3983	}
3984	}
3985
3986	size_t llama_state_seq_get_size(llama_context * ctx, llama_seq_id seq_id) {
3987	return llama_state_seq_get_size_ext(ctx, seq_id, 0);
3988	}
3989
3990	size_t llama_state_seq_get_data(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id) {
3991	return llama_state_seq_get_data_ext(ctx, dst, size, seq_id, 0);
3992	}
3993
3994	size_t llama_state_seq_set_data(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id) {
3995	return llama_state_seq_set_data_ext(ctx, src, size, seq_id, 0);
3996	}
3997
3998	size_t llama_state_seq_get_size_ext(llama_context * ctx, llama_seq_id seq_id, llama_state_seq_flags flags) {
3999	return ctx->state_seq_get_size(seq_id, flags);
4000	}
4001
4002	size_t llama_state_seq_get_data_ext(llama_context * ctx, uint8_t * dst, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) {
4003	ctx->synchronize();
4004
4005	return ctx->state_seq_get_data(seq_id, dst, size, flags);
4006	}
4007	size_t llama_state_seq_set_data_ext(llama_context * ctx, const uint8_t * src, size_t size, llama_seq_id seq_id, llama_state_seq_flags flags) {
4008	ctx->synchronize();
4009
4010	return ctx->state_seq_set_data(seq_id, src, size, flags);
4011	}
4012
4013	size_t llama_state_seq_save_file(llama_context * ctx, const char * filepath, llama_seq_id seq_id, const llama_token * tokens, size_t n_token_count) {
4014	ctx->synchronize();
4015
4016	tryif (true) {
4017	return ctx->state_seq_save_file(seq_id, filepath, tokens, n_token_count);
4018	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
4019	LLAMA_LOG_ERROR("%s: error saving sequence state file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error saving sequence state file: %s\n" , __func__, err.what());
4020	return 0;
4021	}
4022	}
4023
4024	size_t llama_state_seq_load_file(llama_context * ctx, const char * filepath, llama_seq_id dest_seq_id, llama_token * tokens_out, size_t n_token_capacity, size_t * n_token_count_out) {
4025	ctx->synchronize();
4026
4027	tryif (true) {
4028	return ctx->state_seq_load_file(dest_seq_id, filepath, tokens_out, n_token_capacity, n_token_count_out);
4029	} catch (const std::exception & err)if (static const std::exception e, err, error, ex; false) {
4030	LLAMA_LOG_ERROR("%s: error loading sequence state file: %s\n", __func__, err.what())llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: error loading sequence state file: %s\n" , __func__, err.what());
4031	return 0;
4032	}
4033	}
4034
4035	///
4036
4037	int32_t llama_encode(
4038	llama_context * ctx,
4039	llama_batch batch) {
4040	const int ret = ctx->encode(batch);
4041	if (ret != 0) {
4042	LLAMA_LOG_ERROR("%s: failed to encode, ret = %d\n", __func__, ret)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to encode, ret = %d\n" , __func__, ret);
4043	}
4044
4045	return ret;
4046	}
4047
4048	int32_t llama_decode(
4049	llama_context * ctx,
4050	llama_batch batch) {
4051	const int ret = ctx->decode(batch);
4052	if (ret != 0 && ret != 1) {
4053	LLAMA_LOG_ERROR("%s: failed to decode, ret = %d\n", __func__, ret)llama_log_internal(GGML_LOG_LEVEL_ERROR, "%s: failed to decode, ret = %d\n" , __func__, ret);
4054	}
4055
4056	return ret;
4057	}
4058
4059	//
4060	// perf
4061	//
4062
4063	llama_perf_context_data llama_perf_context(const llama_context * ctx) {
4064	llama_perf_context_data data = {};
4065
4066	if (ctx == nullptr) {
4067	return data;
4068	}
4069
4070	data = ctx->perf_get_data();
4071
4072	return data;
4073	}
4074
4075	void llama_perf_context_print(const llama_context * ctx) {
4076	const auto data = llama_perf_context(ctx);
4077
4078	const double t_end_ms = 1e-3 * ggml_time_us();
4079
4080	LLAMA_LOG_INFO("%s: load time = %10.2f ms\n", __func__, data.t_load_ms)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: load time = %10.2f ms\n" , __func__, data.t_load_ms);
4081	LLAMA_LOG_INFO("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval)
4082	__func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);
4083	LLAMA_LOG_INFO("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data .n_eval, 1e3 / data.t_eval_ms * data.n_eval)
4084	__func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n" , __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data .n_eval, 1e3 / data.t_eval_ms * data.n_eval);
4085	LLAMA_LOG_INFO("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval))llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: total time = %10.2f ms / %5d tokens\n" , __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data .n_eval));
4086	LLAMA_LOG_INFO("%s: graphs reused = %10d\n", __func__, data.n_reused)llama_log_internal(GGML_LOG_LEVEL_INFO , "%s: graphs reused = %10d\n" , __func__, data.n_reused);
4087	}
4088
4089	void llama_perf_context_reset(llama_context * ctx) {
4090	ctx->perf_reset();
4091	}
4092
4093	//
4094	// training
4095	//
4096
4097	bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata) {
4098	GGML_UNUSED(tensor)(void)(tensor);
4099	GGML_UNUSED(userdata)(void)(userdata);
4100	return true;
4101	}
4102
4103	void llama_opt_init(struct llama_context * ctx, struct llama_model * model, struct llama_opt_params lopt_params) {
4104	ctx->opt_init(model, lopt_params);
4105	}
4106
4107	void llama_opt_epoch(
4108	struct llama_context * ctx,
4109	ggml_opt_dataset_t dataset,
4110	ggml_opt_result_t result_train,
4111	ggml_opt_result_t result_eval,
4112	int64_t idata_split,
4113	ggml_opt_epoch_callback callback_train,
4114	ggml_opt_epoch_callback callback_eval) {
4115	ctx->opt_epoch(
4116	dataset,
4117	result_train,
4118	result_eval,
4119	idata_split,
4120	callback_train,
4121	callback_eval);
4122	}
4123
4124	//
4125	// ext
4126	//
4127
4128	llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx) {
4129	return ctx->memory_breakdown();
4130	}
4131
4132	llama_context * llama_get_ctx_other(struct llama_context * ctx) {
4133	return ctx->get_cparams().ctx_other;
4134	}