From d30f22422a155c453921eaeb047777c3a3acc99c Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Tue, 11 Aug 2026 22:11:22 +0200 Subject: [PATCH 01/20] Add Muse-Glimmer-30B.IQ4_XS.gguf --- presets/models_24GB_VRAM.ini | 32 +++++++++++++++++++++++++++++++- vendor/llama.cpp | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 576fb62..1a7b7aa 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -165,7 +165,37 @@ direct-io = true no-mmap = true fit = off -[DeepSeek-V4-Flash-0731-UD-Q8_K_XL] +[Muse-Glimmer-30B.IQ4_XS.gguf] +alias = Muse-Glimmer-30B.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Muse-Glimmer-30B\mmproj.Muse-Glimmer-30B.Q8_0.gguf +spec-draft-model = D:\AI\LLM\gguf\Muse-Glimmer-30B-assistant\dflash-Muse-Glimmer-30B-assistant.Q4_0.gguf +mmproj-offload = true +ctx-size = 262144 +parallel = 1 +n-gpu-layers = -1 +cache-type-k = q8_0 +cache-type-v = q8_0 +cache-type-k-draft = q8_0 +cache-type-v-draft = q8_0 +flash-attn = true +jinja = true +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 51200 +min-p = 0.0 +top-p = 0.95 +top-k = 64 +temp = 1.0 +spec-type = draft-dflash,ngram-mod +spec-draft-n-max = 15 +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = dio +fit = off + +[DeepSeek-V4-Flash-0731-UD-Q8_K_XL.gguf] alias = DeepSeek-V4-Flash-0731-UD-Q8_K_XL.gguf model = D:\AI\LLM\gguf\DeepSeek-V4-Flash-0731-GGUF\UD-Q8_K_XL\DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00001-of-00005.gguf ctx-size = 1048576 diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e031d95..030ebb5 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e031d956797f9a10f704fa523fe5fd9b72e9015c +Subproject commit 030ebb558a5820b444a8f836ed5cdd46c9b4bd7a From afc0e4237b6d71d71aac3fde90ec40f15da124b0 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Wed, 12 Aug 2026 15:19:35 +0200 Subject: [PATCH 02/20] Add Muse-Glimmer-30B.IQ3_XXS.gguf preset --- presets/models_16GB_VRAM.ini | 30 ++++++++++++++++++++++++++++++ vendor/llama.cpp | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/presets/models_16GB_VRAM.ini b/presets/models_16GB_VRAM.ini index a971b29..6e96f98 100644 --- a/presets/models_16GB_VRAM.ini +++ b/presets/models_16GB_VRAM.ini @@ -190,3 +190,33 @@ spec-ngram-mod-n-min = 48 spec-ngram-mod-n-max = 64 load-mode = dio fit = off + +[Muse-Glimmer-30B.IQ3_XXS.gguf] +alias = Muse-Glimmer-30B.IQ3_XXS.gguf +model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ3_XXS.gguf +mmproj = D:\AI\LLM\gguf\Muse-Glimmer-30B\mmproj.Muse-Glimmer-30B.Q8_0.gguf +spec-draft-model = D:\AI\LLM\gguf\Muse-Glimmer-30B-assistant\dflash-Muse-Glimmer-30B-assistant.Q4_0.gguf +no-mmproj-offload = true +ctx-size = 131072 +parallel = 1 +n-gpu-layers = -1 +cache-type-k = q5_0 +cache-type-v = q4_1 +cache-type-k-draft = q5_0 +cache-type-v-draft = q4_1 +flash-attn = true +jinja = true +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 51200 +min-p = 0.0 +top-p = 0.95 +top-k = 64 +temp = 1.0 +spec-type = draft-dflash,ngram-mod +spec-draft-n-max = 15 +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = dio +fit = off diff --git a/vendor/llama.cpp b/vendor/llama.cpp index 030ebb5..a4a4c51 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit 030ebb558a5820b444a8f836ed5cdd46c9b4bd7a +Subproject commit a4a4c51f3d40e086b59b73b631b5c43c8fbf4504 From bd877dcc64c7f9b4cdd8962acc7be211f1022810 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sat, 15 Aug 2026 14:37:36 +0200 Subject: [PATCH 03/20] Add Qwen3.8-27B.IQ4_XS.gguf to 24 GB preset --- AGENTS.md | 42 +++++++++++++++++++++++++++++++++++- CHANGELOG.md | 10 +++++++++ presets/README.md | 6 ++++++ presets/models_24GB_VRAM.ini | 32 +++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 9ed4255..0eb061f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -99,6 +99,46 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi GGUF-embedded template; froggeric's README only claims compatibility for Qwen 3.5 / 3.6 variants. +- **`Qwen3.8-27B` also keeps its GGUF-embedded template — do not "fix" the missing pin.** + Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as + Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, + `eos = 248046`), so the tier entry is a clone of the 3.6 one. The template is the + exception: 3.8 embeds a *different, newer* 8952-byte file, not the 7764-byte one the + pin exists to replace. It already applies the fix the pin is for — `preserve_thinking` + now defaults on (`preserve_thinking is undefined or preserve_thinking is true or ...`) + — and it adds `reasoning_effort`, which lives *only* in the model's own template. + Pinning froggeric's file would therefore silently downgrade the template *and* turn + `--reasoning-effort` into a no-op, because that flag only writes a template kwarg + (`common/arg.cpp:3649-3660`) and never reaches the parser. Valid values here are + `xhigh` (default) / `medium` / `low` only — the template calls `raise_exception` on + anything else, so the `"high"` used by the DeepSeek entry is startup-fatal on Qwen. + With `reasoning = on` the default already resolves to `xhigh`, so the entry sets no + `reasoning-effort` key. Tool-call parsing is unaffected either way: the qwen3_coder + XML handler is selected purely on `` + `` / `<|think_off|>`, `developer` role). + +- **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** + 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself + embeds as `general.sampling.temp`, applied at `common/common.cpp:1264` unless the + preset overrides it. The rest of the sampler block (`top-p 0.95`, `top-k 20`, + `min-p 0.0`, `presence-penalty 0`) is unchanged from the 3.6 entries. Qwen 3.8's + non-thinking mode wants a different set (`temp 0.7`, `top-p 0.8`, + `presence-penalty 1.5`); the preset does not cover it because `reasoning = on`. + +- **Qwen 3.8's MTP head is multi-step trained, so `spec-draft-n-max = 3` is a measured + peak rather than an inherited default.** A day-0 `n_max` sweep of 2/3/4/6 on this model + put the maximum at 3 — acceptance falls monotonically with depth, but through 3 the + extra tokens per iteration win. This overturns the Qwen 3.6 rule of thumb that 2 was + optimal. 3 also happens to be the upstream default (`common/common.h:325`). Note the + cost: `draft-mtp` sets `n_rs_seq = spec-draft-n-max` (`common/common.cpp:1699`), which + multiplies the recurrent-state buffer by `1 + n_max` — ~150 MiB becomes ~600 MiB at 3. + Both `Qwen3.8-27B` and `Qwen3.6-27B` carry `blk.64` (the MTP head) at `Q4_0` in the + local IQ4_XS files; a 4-bit MTP head is reported to collapse acceptance to 0% on this + model family, so check the server's acceptance rate before trusting the speedup — + the fix would be a re-quant keeping `blk.64` at `Q5_K` or above, not a preset change. + - **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** This is Google's fixed official template as aligned by upstream (#21704) — the exact file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler @@ -122,7 +162,7 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi derivatives: arch `qwen35`, and their tokenizers are byte-identical to stock Qwen3.6-27B (248320 tokens, same merges, `eos = 248046`) right down to the same 7764-byte embedded template — which is exactly the upstream template the pin exists to replace. `general.sampling.temp = 1.0` is - embedded in both GGUFs and applied at `common/common.cpp:1194`, so `temp` has to be pinned in the + embedded in both GGUFs and applied at `common/common.cpp:1264`, so `temp` has to be pinned in the preset or generation runs at 1.0. The presets use `0.6` to match the sibling Qwen 3.6 entries; Prism's own card benchmarks at `0.7`. Unlike the DSpark sidecar below, both weight files are mainline-packed (`Q2_0` at `QK2_0 64`, `Q1_0` at `QK1_0 128`) and load without a tensor-offset diff --git a/CHANGELOG.md b/CHANGELOG.md index a045649..4c1b2a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.41.0] - 2026-08-15 + +### Added +- [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier +- [Documentation] Document why Qwen3.8-27B keeps its GGUF-embedded chat template + +### Fixed +- [Documentation] Correct the common.cpp line reference for the embedded sampling temperature + + ## [1.40.0] - 2026-08-05 ### Added diff --git a/presets/README.md b/presets/README.md index f54cf7a..4f3dfe1 100644 --- a/presets/README.md +++ b/presets/README.md @@ -77,6 +77,12 @@ The section header (e.g. `[gemma-4-31B-it.IQ4_XS.gguf]`) is the model name clien > If you cloned without `--recurse-submodules`, run `git submodule update --init` > first — otherwise startup fails with a missing-file error. > +> `Qwen3.8-27B` deliberately does **not** pin it. Qwen 3.8 embeds a different, newer +> template than the one the pin replaces: it already defaults `preserve_thinking` to +> true and adds `reasoning_effort` (`xhigh`/`medium`/`low`), which exists only in the +> model's own template. Pinning the vendored file here would silently turn +> `--reasoning-effort` into a no-op. +> > All `gemma-4-*` entries set `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja` — > the official Google template bundled with llama.cpp itself, kept in lock-step with > its built-in Gemma 4 parser. The same repo-root launch caveat applies. diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 1a7b7aa..e74f723 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -99,6 +99,38 @@ spec-ngram-mod-n-max = 64 load-mode = dio fit = off +[Qwen3.8-27B.IQ4_XS.gguf] +alias = Qwen3.8-27B.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Qwen3.8-27B\Qwen3.8-27B.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.8-27B\mmproj.Qwen3.8-27B.BF16.gguf +mmproj-offload = true +image-min-tokens = 1024 +ctx-size = 262144 +parallel = 1 +n-gpu-layers = -1 +cache-type-k = q4_0 +cache-type-v = q4_0 +cache-type-k-draft = q4_0 +cache-type-v-draft = q4_0 +flash-attn = true +reasoning = on +jinja = true +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 51200 +min-p = 0.0 +top-p = 0.95 +top-k = 20 +temp = 1.0 +presence-penalty = 0 +spec-type = draft-mtp,ngram-mod +spec-draft-n-max = 3 +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = dio +fit = off + [Qwen3.6-35B-A3B.IQ4_XS.gguf] alias = Qwen3.6-35B-A3B.IQ4_XS.gguf model = D:\AI\LLM\gguf\Qwen3.6-35B-A3B\Qwen3.6-35B-A3B.IQ4_XS.gguf From 3c53c4020d837d31dee7e40cae5ef3a0a0067824 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sat, 15 Aug 2026 15:36:43 +0200 Subject: [PATCH 04/20] Switch Qwen3.8-27B to the Q8_0 projector and drop heretic-v2 --- AGENTS.md | 27 +++++++++++++++++++++++++-- CHANGELOG.md | 12 ++++++++++++ presets/models_24GB_VRAM.ini | 35 +---------------------------------- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0eb061f..0b23c91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -72,7 +72,8 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi `clip.cpp:1502-1506` warns on every load because upstream needs >= 1024 tokens (1024x1024 px) for grounding (#16842). The key raises a floor only: images already above 1024 tokens are unchanged, smaller ones get upscaled, which costs context and CLIP time on the CPU because the 16 GB tier - sets `no-mmproj-offload = true`. Applies to the `qwen3vl_merger` entries (Qwen3.6 and + sets `no-mmproj-offload = true`; the 24 GB `Qwen3.8-27B` entry offloads CLIP to the GPU and + pays it there instead. Applies to the `qwen3vl_merger` entries (Qwen3.6, Qwen3.8 and Ternary-Bonsai); gemma-4 uses a different projector and must not get this key. - **`mmproj-offload = true` fails silently at startup on a saturated GPU.** CLIP's warmup @@ -102,7 +103,7 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi - **`Qwen3.8-27B` also keeps its GGUF-embedded template — do not "fix" the missing pin.** Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, - `eos = 248046`), so the tier entry is a clone of the 3.6 one. The template is the + `eos = 248046`), so the tier entry is a near-clone of the 3.6 one. The template is the exception: 3.8 embeds a *different, newer* 8952-byte file, not the 7764-byte one the pin exists to replace. It already applies the fix the pin is for — `preserve_thinking` now defaults on (`preserve_thinking is undefined or preserve_thinking is true or ...`) @@ -139,6 +140,28 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi model family, so check the server's acceptance rate before trusting the speedup — the fix would be a re-quant keeping `blk.64` at `Q5_K` or above, not a preset change. +- **`Qwen3.8-27B` is the only 24 GB Qwen entry on a `Q8_0` projector instead of `BF16`.** + 600 MiB rather than 888 MiB of VRAM, and that saving is what keeps `mmproj-offload = true` + affordable at `ctx-size = 262144`: the entry lands at ~19.95 GiB of ~22.6 GiB usable, just + under the Qwen3.6-27B entry's ~20.23 GiB, which leaves room for the CLIP compute buffer. + Spending the saving elsewhere is what breaks it — raising the KV cache to `q5_0` K / `q4_1` V + costs ~0.80 GiB at this context and pushes the total above the 3.6 entry, into the + silent-OOM window described above. Quality is not the tradeoff: Qwen ships this family's + projector as FP16 *and* `Q8_0` officially, and only 83 of the file's 110 weight tensors are + actually 8-bit — every `ffn_down` stays `F16`. + +- **Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from `Qwen/Qwen3.8-27B-FP8`.** BF16 is + this model's native precision and the FP8 repo is a derived, post-training artifact (HF model + tree: base model `Qwen3.8-27B`, "Quantized"), so it is already lossy — its card claims only + "nearly identical" metrics. Quantizing from it would fit the quantizer and the imatrix to + degraded weights. This is the opposite of DeepSeek-V3/V4, which were *trained* in FP8, making + their FP8 checkpoint the original and its dequant to BF16 exact; `convert_hf_to_gguf.py:156` + (`--fp8-as-q8`) exists for that case, not this one. The mmproj is the one exception where the + source does not matter: the FP8 repo leaves the whole vision tower unquantized (0 of 333 + `model.visual.*` tensors carry `weight_scale_inv`), so it is byte-identical either way. The + MTP head is not — `mtp.layers.0`'s attention and MLP projections are FP8 there, so any + re-quant raising `blk.64` above 4-bit must also come from the BF16 repo. + - **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** This is Google's fixed official template as aligned by upstream (#21704) — the exact file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c1b2a9..730ed87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.42.0] - 2026-08-15 + +### Added +- [Documentation] Document the Qwen3.8-27B projector choice and its quantization source + +### Changed +- [Presets] Switch the Qwen3.8-27B vision projector to Q8_0 + +### Removed +- [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier + + ## [1.41.0] - 2026-08-15 ### Added diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index e74f723..4bc8687 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -102,7 +102,7 @@ fit = off [Qwen3.8-27B.IQ4_XS.gguf] alias = Qwen3.8-27B.IQ4_XS.gguf model = D:\AI\LLM\gguf\Qwen3.8-27B\Qwen3.8-27B.IQ4_XS.gguf -mmproj = D:\AI\LLM\gguf\Qwen3.8-27B\mmproj.Qwen3.8-27B.BF16.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.8-27B\mmproj.Qwen3.8-27B.Q8_0.gguf mmproj-offload = true image-min-tokens = 1024 ctx-size = 262144 @@ -164,39 +164,6 @@ spec-ngram-mod-n-max = 64 load-mode = dio fit = off -[Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf] -alias = Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf -model = D:\AI\LLM\gguf\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf -mmproj = D:\AI\LLM\gguf\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved\mmproj.Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.BF16.gguf -mmproj-offload = true -ctx-size = 262144 -parallel = 1 -n-gpu-layers = -1 -cache-type-k = q5_0 -cache-type-v = q4_1 -cache-type-k-draft = q5_0 -cache-type-v-draft = q4_1 -flash-attn = true -reasoning = on -jinja = true -chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja -kv-unified = true -ctx-checkpoints = 32 -cache-ram = 51200 -min-p = 0.0 -top-p = 0.95 -top-k = 20 -temp = 0.6 -presence-penalty = 0 -spec-type = draft-mtp,ngram-mod -spec-draft-n-max = 3 -spec-ngram-mod-n-match = 24 -spec-ngram-mod-n-min = 48 -spec-ngram-mod-n-max = 64 -direct-io = true -no-mmap = true -fit = off - [Muse-Glimmer-30B.IQ4_XS.gguf] alias = Muse-Glimmer-30B.IQ4_XS.gguf model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ4_XS.gguf From b5ad8240eda80a28e06bb089e0ff26c969150cb1 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sat, 15 Aug 2026 15:38:50 +0200 Subject: [PATCH 05/20] Consolidate unreleased CHANGELOG 1.42.0 into 1.41.0 --- CHANGELOG.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 730ed87..fdbe9b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,24 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.42.0] - 2026-08-15 +## [1.41.0] - 2026-08-15 ### Added +- [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier +- [Documentation] Document why Qwen3.8-27B keeps its GGUF-embedded chat template - [Documentation] Document the Qwen3.8-27B projector choice and its quantization source -### Changed -- [Presets] Switch the Qwen3.8-27B vision projector to Q8_0 - ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier - -## [1.41.0] - 2026-08-15 - -### Added -- [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier -- [Documentation] Document why Qwen3.8-27B keeps its GGUF-embedded chat template - ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature From 7aa62927fcb95be3db0e9fcb7942ab9f9a7a0ee9 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sat, 15 Aug 2026 21:14:38 +0200 Subject: [PATCH 06/20] Optimize Muse-Glimmer-30B 24 GB preset for 262144 usable context --- AGENTS.md | 81 +++++++++++++++++++++++++++++++++--- CHANGELOG.md | 10 +++++ presets/models_24GB_VRAM.ini | 13 +++--- 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0b23c91..1bc027e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -290,11 +290,12 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi `function_calls`-absent test at `:2105`. Unsloth's additionally restores `reasoning_content` on tool calls, which the official template drops. Unlike gemma-4 there is no outdated-template rewrite path for `deepseek4` — detection is all-or-nothing, and a miss degrades to the generic - autoparser rather than being repaired. `reasoning_effort` has no CLI flag - (`server-common.cpp:1089-1095` honours only the literal `"none"`), so the only route is - `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); left unset the template - defaults it to `none` and emits no effort block at all, and `reasoning = off` voids it - entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` + autoparser rather than being repaired. The entry carries the effort as + `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); `reasoning-effort = high` is + equivalent and simpler — `common/arg.cpp:3650` writes the same template kwarg, and the level + `xhigh` is listed in its help text. Only the *request* field special-cases the literal `"none"` + (`server-common.cpp:1296-1304`). Left unset the template defaults it to `none` and emits no + effort block at all, and `reasoning = off` voids it entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` (`common.h:631`, despite the help text saying `auto`), and `none` leaks `` into `content`. @@ -312,6 +313,76 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi after ~2500 tokens (#26554). The regression that broke spec decoding on this arch (#26576, a 2D `wo_a` in `dflash.cpp` after #26531) is fixed by #26577 at `b10269`. +- **`ctx-size` above the GGUF's `context_length` is dead VRAM unless `override-kv` lifts it too.** + `llama-context.cpp:131` never clamps `n_ctx`, so the KV cache really is allocated at the + requested size — but the server then caps every slot at `n_ctx_train` + (`tools/server/server-context.cpp:1201-1203`, applied as `slot.n_ctx = n_ctx_slot` at `:1255`) + and rejects any larger request outright at `:3100` / `:3111`. Both Muse Glimmer GGUFs ship + `muse-glimmer.context_length = 131072`, so the 24 GB entry's former `ctx-size = 262144` + allocated 262144 cells while no request could exceed 131072 — ~884 MiB of unreachable VRAM. + `override-kv = muse-glimmer.context_length=int:262144` raises `n_ctx_train` and is the only + lever; Meta documents 131072 as the default and 262144 as the maximum, so this is the + vendor-sanctioned ceiling, not an extrapolation hack. The startup line + `the slot context (...) exceeds the training context of the model (...) - capping` is expected + here (the pool is 524288 for two 262144 slots); what matters is + `initializing, n_slots = 2, n_ctx_slot = 262144`. If that reads 131072 the override did not + take. It also sets `n_ctx_orig_yarn = 262144` (`src/llama-model.cpp:1180`), inert at + `freq_scale 1`. + +- **Muse Glimmer has no architectural positional ceiling, so never add RoPE scaling.** Meta's + `config.json` sets `layer_rope_theta = 0` on all 13 `full_attention` layers (NoPE) and + `sliding_window = 2048` on the other 39; the GGUF's `sliding_window_pattern` confirms the + `[true, true, true, false]` x13 split. RoPE therefore never sees a relative position above 2048 + at any context length — 131072 is a training length, not a limit, which is why Meta writes + "131,072+". Because the GGUF carries no `rope.scaling.*` keys the type defaults to `linear` with + `freq_scale 1` (`src/llama-model.cpp:1187-1189`), so a user `--rope-scale` is *applied*, not + ignored, and would compress local resolution on the SWA layers for no benefit. The + `--rope-scaling yarn` recipes circulating on the HF model page are wrong; only their + `--override-kv` half is load-bearing. + +- **Never set `swa-full` or `context-shift` on a Muse Glimmer entry.** `swa-full` collapses the + 4608-cell SWA cache to the full `n_ctx` by the mechanism documented for DeepSeek above — a ~114x + blowup at `ctx-size = 524288`. `context-shift` is the more dangerous one, and it is the exact + inverse of the `deepseek4` case: `llama_hparams::has_rope()` knows only about `router_layer` + (`src/llama-hparams.cpp:287-295`), so a K-shift rotates 128 dims on all 52 layers including the + 13 NoPE ones, while `get_can_shift()` returns true. Silent corruption rather than a refusal. It + is dormant only because `ctx_shift` defaults false (`common/common.h:561`) and loading an mmproj + force-disables it (`server-context.cpp:1163-1171`); dropping the mmproj would make it reachable. + +- **`spec-draft-n-max = 15` is `block_size - 1`, and the absent `chat-template-file` and + `reasoning` keys are both deliberate.** The drafter carries `dflash.block_size = 16` and spends + block position 0 on the committed anchor, so 15 is the maximum legal draft + (`common/speculative.cpp:970-978`, no clamp and no warning); the upstream default is 3 + (`common/common.h:325`), so leaving it unset discards 80% of a block that is decoded in one + forward pass regardless. Judge it by `mean len` in the slot timings + (`1 + n_accepted / n_verif_steps`, `server-context.cpp:620`), never by `draft acceptance` — that + ratio is a percentage of *drafted* tokens and is meaningless for block diffusion, where + unaccepted drafts cost nothing. Measured ~3.4 tokens per target pass at short context, ~3.0 at + 4.3k. No template may be pinned: `common/chat.cpp:3276` selects the native Muse Glimmer handler + on `` + `<|eom|>` in the template source, and there is no bundled file to + pin. `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads + `enable_thinking`; reasoning here is the structural ` to=self<|message|>...<|eom|>` channel, + gated on `reasoning_format` (`chat.cpp:3141`). The template already defaults to + `Reasoning strength: high` (`models/templates/muse-glimmer.jinja:84`), which matches Meta's + recommendation, and `--reasoning-effort` reaches it only through the + `reasoning_effort` -> `reasoning_strength` alias at `common/jinja/caps.cpp:29-33`. + `--reasoning-budget` is inert: the handler sets no `thinking_end_tags` + (`server-common.cpp:1343`). + +- **Compute buffers dominate this entry's headroom and cannot be derived from KV arithmetic.** + Measured on a 24463 MiB card at `ctx-size = 524288` / `parallel = 2` / `q5_0` K + `q4_1` V: + 13488.92 MiB target weights, 1371.40 draft, 1956.60 CLIP, 2271.12 KV (2184.00 non-SWA + + 57.59 SWA + 29.53 draft) and **2655.59 MiB of compute buffers** (1135.67 target + 803.03 + spec-context + 407.62 draft + 309.27 CLIP) — 21743.63 MiB total, leaving ~1.0 GiB free. The + compute term is larger than the whole KV cache and grew 1070.67 -> 1135.67 MiB once a real + request arrived, so budget it explicitly instead of sizing `ctx-size` off KV alone. Note the + SWA cache scales with `parallel`, not `ctx-size` + (`PAD(min(n_ctx, n_swa * n_seq_max + n_ubatch), 256)` = 4608 cells), so raising `parallel` + is cheap while raising `ctx-size` is not. `mmproj-offload = true` survives this margin (CLIP + warmup reserves 309.27 MiB and real images decode), but it is the first thing to disable if the + margin shrinks — see the silent-OOM note above. Context checkpoints are host-side, ~38.8 MiB + each at `max = 32`. + **ngram-mod speculative decoding** (`--spec-type ngram-mod`): model-agnostic, works on any model. - All models: `spec-ngram-mod-n-match = 24`, `spec-ngram-mod-n-min = 48`, `spec-ngram-mod-n-max = 64` (matches the struct defaults in `common/common.h:329-337` and what `--spec-default` produces diff --git a/CHANGELOG.md b/CHANGELOG.md index fdbe9b6..e9e8c39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier - [Documentation] Document why Qwen3.8-27B keeps its GGUF-embedded chat template - [Documentation] Document the Qwen3.8-27B projector choice and its quantization source +- [Documentation] Document that ctx-size above the GGUF context_length is dead VRAM without override-kv +- [Documentation] Document Muse-Glimmer-30B RoPE, swa-full, and context-shift behavior +- [Documentation] Document the Muse-Glimmer-30B DFlash drafter, chat template, and reasoning strength defaults +- [Documentation] Document the measured Muse-Glimmer-30B 24 GB compute buffer and VRAM breakdown + +### Changed +- [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 +- [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 +- [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature +- [Documentation] Correct the claim that reasoning_effort has no CLI flag ## [1.40.0] - 2026-08-05 diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 4bc8687..afc9165 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -170,13 +170,14 @@ model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ4_XS.gguf mmproj = D:\AI\LLM\gguf\Muse-Glimmer-30B\mmproj.Muse-Glimmer-30B.Q8_0.gguf spec-draft-model = D:\AI\LLM\gguf\Muse-Glimmer-30B-assistant\dflash-Muse-Glimmer-30B-assistant.Q4_0.gguf mmproj-offload = true -ctx-size = 262144 -parallel = 1 +ctx-size = 524288 +override-kv = muse-glimmer.context_length=int:262144 +parallel = 2 n-gpu-layers = -1 -cache-type-k = q8_0 -cache-type-v = q8_0 -cache-type-k-draft = q8_0 -cache-type-v-draft = q8_0 +cache-type-k = q5_0 +cache-type-v = q4_1 +cache-type-k-draft = q5_0 +cache-type-v-draft = q4_1 flash-attn = true jinja = true kv-unified = true From 71d4fcf8fb6e3a364b8d954b28d69ec31d0a06b1 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sat, 15 Aug 2026 21:22:32 +0200 Subject: [PATCH 07/20] Bump Qwen chat template to v22 and pin it on Qwen3.8-27B --- AGENTS.md | 82 +++++++++++++++++++++----------- CHANGELOG.md | 7 ++- presets/README.md | 18 +++---- presets/models_16GB_8GB_VRAM.ini | 3 ++ presets/models_16GB_VRAM.ini | 6 +++ presets/models_24GB_VRAM.ini | 3 ++ vendor/Qwen-Fixed-Chat-Templates | 2 +- 7 files changed, 84 insertions(+), 37 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1bc027e..dc25156 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,45 +80,73 @@ See `presets/README.md` for the user-facing quick-start; notes below are for edi compute buffer OOMs but the server keeps running — only image requests error at generation time. Set `false` on tiers where LLM + KV already saturate VRAM. -- **All Qwen 3.6 entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** +- **All Qwen 3.6, Qwen 3.8 and Bonsai entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** Required, *not* redundant with `jinja = true` — `chat-template-file` *replaces* the GGUF-embedded template entirely (`vendor/llama.cpp/common/arg.cpp:3142`, `params.chat_template = read_file(value)`). The upstream embedded template has documented issues with tool calls, role handling, `` block rendering, agentic loops, and llama.cpp KV-prefix cache stability; the vendored template fixes all of them (full list in `vendor/Qwen-Fixed-Chat-Templates/README.md`). - Since v19 the template is a single unified file covering both Qwen 3.5 and 3.6 - variants (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under + Since v19 the template is a single unified file, and since v22 it covers Qwen 3.5, + 3.6 *and* 3.8 (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under `archive/`). The template adds a `<|think_on|>` / `<|think_off|>` toggle, and - v19 defaults `preserve_thinking` to `true` (past `` blocks are kept + defaults `preserve_thinking` to `true` (past `` blocks are kept chronologically for 100% KV prefix cache stability and agentic reasoning continuity). To strip past `` blocks instead, set `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower - KV cache hit rate. Path is repo-relative, so `llama-server` must be launched - from the repo root — `read_file()` resolves against the process CWD, not the - INI file's directory. `Qwen3-Coder-Next` entries deliberately keep their - GGUF-embedded template; froggeric's README only claims compatibility for - Qwen 3.5 / 3.6 variants. - -- **`Qwen3.8-27B` also keeps its GGUF-embedded template — do not "fix" the missing pin.** + KV cache hit rate. v22 also honours `preserve_reasoning`, so `--reasoning-preserve` + (`common/arg.cpp:3677-3689`) works as the CLI equivalent. Path is repo-relative, + so `llama-server` must be launched from the repo root — `read_file()` resolves + against the process CWD, not the INI file's directory. `Qwen3-Coder-Next` entries + deliberately keep their GGUF-embedded template; froggeric's README claims + compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. + +- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry does not.** + v22 added Qwen 3.8's reasoning-effort steering but gates it on nothing — the default + resolves to `xhigh` for *every* model the template serves, injecting a ~45-token + "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. + Qwen 3.6 has no trained notion of the concept, so the entries pin `medium`, the one + level for which v22 emits no instruction text at all. Qwen 3.8 *is* trained on it and + its own template defaults to `xhigh`, so that entry leaves the key unset and inherits + the same default it had before the pin. `--reasoning-effort` writes only a template + kwarg (`common/arg.cpp:3650-3660`), which a request can still override + (`tools/server/server-common.cpp:1296-1303`). Unlike the GGUF-embedded 3.8 template, + v22 never raises on an unknown level — `high` is aliased to `xhigh` and anything else + falls back to it. + +- **`Qwen3.8-27B` pins the template too — v22 removed the reason it used to be the exception.** Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, - `eos = 248046`), so the tier entry is a near-clone of the 3.6 one. The template is the - exception: 3.8 embeds a *different, newer* 8952-byte file, not the 7764-byte one the - pin exists to replace. It already applies the fix the pin is for — `preserve_thinking` - now defaults on (`preserve_thinking is undefined or preserve_thinking is true or ...`) - — and it adds `reasoning_effort`, which lives *only* in the model's own template. - Pinning froggeric's file would therefore silently downgrade the template *and* turn - `--reasoning-effort` into a no-op, because that flag only writes a template kwarg - (`common/arg.cpp:3649-3660`) and never reaches the parser. Valid values here are - `xhigh` (default) / `medium` / `low` only — the template calls `raise_exception` on - anything else, so the `"high"` used by the DeepSeek entry is startup-fatal on Qwen. - With `reasoning = on` the default already resolves to `xhigh`, so the entry sets no - `reasoning-effort` key. Tool-call parsing is unaffected either way: the qwen3_coder - XML handler is selected purely on `` + `` / `<|think_off|>`, `developer` role). + `eos = 248046`), but it embeds a *different, newer* 8952-byte template, not the + 7764-byte one shared by Qwen3.6-27B and both Bonsai variants. That newer file already + defaults `preserve_thinking` on and adds `reasoning_effort`, which is why the entry + shipped unpinned until v22 supported 3.8. Three defects justify pinning anyway, all + reproducible by rendering the embedded template directly: tool calls whose `arguments` + arrive as a JSON *string* (what most OpenAI-compatible clients send) abort with + `Can only get item pairs from a mapping`; history carrying reasoning inside `content` + rather than `reasoning_content` renders a duplicate blank `\n\n` ahead + of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` + accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, + `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. + v22 handles all three and its `xhigh` instruction text is byte-identical to the + official one, so `reasoning = on` with no `reasoning-effort` key reproduces the + pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is + selected purely on `` + `` block. The pin + additionally brings froggeric's agentic extras (two-tier tool-error escalation, + `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). + +- **v22 fixed two v19 deviations from the official Qwen prompt format, so the bump changed + every pinned entry, not just the new 3.8 one.** v19 serialized `` entries + *unwrapped* (`{"description": ..., "name": ..., "parameters": ...}`); v22 emits the + wrapped OpenAI form (`{"function": {...}, "type": "function"}`), which is what + Qwen3.6-27B's own 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 + was the outlier. v19 also rendered `\n` before assistant content where the + official templates and llama.cpp's own generation prompt use `\n\n` + (`common/chat.cpp:1163`). Both are corrections, but they change prompt bytes, so the + bump invalidates existing KV prefix caches once; `reasoning-effort = medium` suppresses + only the new steering paragraph and does not restore v19 output. - **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself diff --git a/CHANGELOG.md b/CHANGELOG.md index e9e8c39..3e02244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier -- [Documentation] Document why Qwen3.8-27B keeps its GGUF-embedded chat template +- [Presets] Pin chat-template-file on the Qwen3.8-27B entry +- [Presets] Pin reasoning-effort to medium on all Qwen 3.6 and Bonsai entries +- [Documentation] Document why Qwen3.8-27B now pins the vendored chat template +- [Documentation] Document the reasoning-effort steering added by chat template v22 +- [Documentation] Document the tools serialization and think-tag spacing fixes in chat template v22 - [Documentation] Document the Qwen3.8-27B projector choice and its quantization source - [Documentation] Document that ctx-size above the GGUF context_length is dead VRAM without override-kv - [Documentation] Document Muse-Glimmer-30B RoPE, swa-full, and context-shift behavior @@ -17,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document the measured Muse-Glimmer-30B 24 GB compute buffer and VRAM breakdown ### Changed +- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 - [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V diff --git a/presets/README.md b/presets/README.md index 4f3dfe1..e9fd11b 100644 --- a/presets/README.md +++ b/presets/README.md @@ -69,19 +69,21 @@ The section header (e.g. `[gemma-4-31B-it.IQ4_XS.gguf]`) is the model name clien > See `llama-server --help` for all flags. > [!IMPORTANT] -> All `Qwen3.6-*` and `Ternary-Bonsai-27B` entries set `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`, +> All `Qwen3.6-*`, `Qwen3.8-*`, `Ternary-Bonsai-27B` and `Bonsai-27B` entries set +> `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`, > overriding the buggy template embedded in the GGUF. The vendored template is a -> single unified file that handles both Qwen 3.5 and 3.6 variants; Ternary Bonsai 27B is a -> Qwen3.6-27B derivative with a byte-identical tokenizer and embedded template. The path is +> single unified file that handles Qwen 3.5, 3.6 and 3.8 variants; both Bonsai models are +> Qwen3.6-27B derivatives with a byte-identical tokenizer and embedded template. The path is > repo-relative, so launch `llama-server` from the repository root (as the examples above do). > If you cloned without `--recurse-submodules`, run `git submodule update --init` > first — otherwise startup fails with a missing-file error. > -> `Qwen3.8-27B` deliberately does **not** pin it. Qwen 3.8 embeds a different, newer -> template than the one the pin replaces: it already defaults `preserve_thinking` to -> true and adds `reasoning_effort` (`xhigh`/`medium`/`low`), which exists only in the -> model's own template. Pinning the vendored file here would silently turn -> `--reasoning-effort` into a no-op. +> The Qwen 3.6 and Bonsai entries additionally set `reasoning-effort = medium`. The vendored +> template defaults to `xhigh`, which prepends a "Reasoning effort is set to xhigh..." +> paragraph to the system prompt; `medium` is the one level that emits no instruction text, +> and Qwen 3.6 has no trained notion of the concept. `Qwen3.8-27B` leaves the key unset +> because it *is* trained on it and `xhigh` is its own template's default too. Clients can +> still override per request via the OpenAI `reasoning_effort` field. > > All `gemma-4-*` entries set `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja` — > the official Google template bundled with llama.cpp itself, kept in lock-step with diff --git a/presets/models_16GB_8GB_VRAM.ini b/presets/models_16GB_8GB_VRAM.ini index 7ae3b58..e8a0008 100644 --- a/presets/models_16GB_8GB_VRAM.ini +++ b/presets/models_16GB_8GB_VRAM.ini @@ -54,6 +54,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -88,6 +89,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -122,6 +124,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 diff --git a/presets/models_16GB_VRAM.ini b/presets/models_16GB_VRAM.ini index 6e96f98..022e842 100644 --- a/presets/models_16GB_VRAM.ini +++ b/presets/models_16GB_VRAM.ini @@ -16,6 +16,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -49,6 +50,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -82,6 +84,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -115,6 +118,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -147,6 +151,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -177,6 +182,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index afc9165..c314b12 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -83,6 +83,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -115,6 +116,7 @@ cache-type-v-draft = q4_0 flash-attn = true reasoning = on jinja = true +chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -148,6 +150,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 diff --git a/vendor/Qwen-Fixed-Chat-Templates b/vendor/Qwen-Fixed-Chat-Templates index c31fd39..9f14778 160000 --- a/vendor/Qwen-Fixed-Chat-Templates +++ b/vendor/Qwen-Fixed-Chat-Templates @@ -1 +1 @@ -Subproject commit c31fd393e531dbacd92b6deb99a2037cc949f950 +Subproject commit 9f14778c92c3b5ed3e0738085694c0d3452802dd From 8c7f6d25eb75ce5ff1ce99d3292fc42477eca0fd Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sun, 16 Aug 2026 21:13:28 +0200 Subject: [PATCH 08/20] Move preset rationale from AGENTS.md into docs/presets.md and docs/model_tuning.md AGENTS.md was 443 lines, 88% of it the Presets knowledge base, loaded into every session regardless of task. The rationale is now read on demand via a Reference table, with an always-on Traps section carrying the prohibitions whose violation is a silent OOM or silent corruption. Content moved verbatim; verified line-for-line against the previous revision. --- AGENTS.md | 414 +++---------------------------------------- CHANGELOG.md | 2 + docs/model_tuning.md | 340 +++++++++++++++++++++++++++++++++++ docs/presets.md | 84 +++++++++ 4 files changed, 452 insertions(+), 388 deletions(-) create mode 100644 docs/model_tuning.md create mode 100644 docs/presets.md diff --git a/AGENTS.md b/AGENTS.md index dc25156..c53d6e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md -This file provides guidance to coding agents (Claude Code, OpenCode) when working with code in this repository. +Canonical agent-instruction file for this repository. Both Claude Code (via the `@AGENTS.md` import in `CLAUDE.md`) and OpenCode (which reads `AGENTS.md` natively) load this file. It carries the always-on rules; deep reference documentation lives under `docs/` and is read on demand, not loaded into context (see *Reference* at the end). ## What this is @@ -36,408 +36,46 @@ Binaries land in `./vendor/llama.cpp/build/bin/Release/`. Conda env `llama.cpp` - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. - **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. If startup fails reading the script after a rebuild, check whether upstream moved `tools/server/bench/speed-bench/` (same failure mode as the `gguf_dump.py` note above). - **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. -- **`load-mode = dio` does not enable DirectIO on Windows — it only disables mmap.** The Win32 `llama_file::impl` ctor takes `use_direct_io` as `[[maybe_unused]]` and just calls `ggml_fopen` (`vendor/llama.cpp/src/llama-mmap.cpp:86-95`); `FILE_FLAG_NO_BUFFERING` is never set and `read_alignment()` stays 1 (`:391`), so the loader's async staging buffers are 4 x 1 MiB of pinned host memory instead of the 4 x 64 MiB the aligned path would use (`src/llama-model-loader.cpp:1418`, `:1427`). `has_direct_io()` nevertheless returns a hardcoded `true` on Windows (`:173-175`). Net effect of `dio` on this platform: buffered reads, no mmap, and zero VRAM cost — it is never implicated in a CUDA OOM. Keep the key for the deprecation-warning reason documented under Presets, but do not reason about page-cache behaviour from it. ## Presets -VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, -`presets/models_16GB_8GB_VRAM.ini` (dual-GPU). -See `presets/README.md` for the user-facing quick-start; notes below are for editing. +VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, `presets/models_16GB_8GB_VRAM.ini` (dual-GPU). -- **Only `models_16GB_8GB_VRAM.ini` pins devices.** It sets `split-mode`, `main-gpu`, and - `tensor-split` in its `[*]` section; the other two tiers set none of the three, so on a host with - more than one CUDA device llama.cpp's default `split-mode = layer` spreads every entry across - *all* visible GPUs — the tier name is then a floor, not a cap. Pin with `CUDA_VISIBLE_DEVICES` - before launch, not `--device`: in router mode each child's argv is rebuilt from the preset - (`inst.meta.update_args`), so a parent `--device` never reaches the child, while the environment - is copied into every child (`tools/server/server-models.cpp:802`). Pinning is a throughput - decision, not only a memory one — `Ternary-Bonsai-27B-Q2_g64.gguf` measured 56.5 t/s tg and - 1418 t/s pp on one 16 GB card versus 36.5 t/s and 998 t/s spread over a 16 GB plus an 8 GB card. - `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and - therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. +`presets/README.md` is the user-facing quick-start. For editing, the cross-model rules are in `docs/presets.md` and the per-model rationale with its measured numbers is in `docs/model_tuning.md`. -- **All entries use `load-mode = dio`; never pair `direct-io` with `no-mmap` again.** Both spellings - are deprecated, and they write the *same* mutually exclusive enum — `--no-mmap` sets - `LLAMA_LOAD_MODE_NONE` (`common/arg.cpp:2594`) while `--direct-io` sets - `LLAMA_LOAD_MODE_DIRECT_IO` (`:2603`) — so setting both means only whichever is parsed last wins. - Which one that is is *not* the INI order: `common/preset.cpp` emits `opt.args.back()` while - iterating an unordered map. It happened to resolve to DirectIO, but a container-order change would - silently downgrade to `NONE`, dropping DirectIO *and* mmap for plain buffered reads. `dio` is the - single equivalent of the old pair and also removes two deprecation warnings per launch. Valid - values are `none`, `mmap`, `mlock`, `mmap+mlock`, `dio` (`arg.cpp:2615-2619`) — anything else - throws at startup. +## Traps -- **Qwen-VL entries pin `image-min-tokens = 1024`.** `clip.cpp:1500` sets the per-image token limits - to `(8, 4096)`, so the default *minimum* is 8 tokens — 8192 px at `merge = 2` / `patch = 16` — and - `clip.cpp:1502-1506` warns on every load because upstream needs >= 1024 tokens (1024x1024 px) for - grounding (#16842). The key raises a floor only: images already above 1024 tokens are unchanged, - smaller ones get upscaled, which costs context and CLIP time on the CPU because the 16 GB tier - sets `no-mmproj-offload = true`; the 24 GB `Qwen3.8-27B` entry offloads CLIP to the GPU and - pays it there instead. Applies to the `qwen3vl_merger` entries (Qwen3.6, Qwen3.8 and - Ternary-Bonsai); gemma-4 uses a different projector and must not get this key. +Prohibitions that cause a silent OOM, silent corruption, or a startup abort. Each is stated here without rationale so it is always in context; read the linked section before acting on one. These lines are a deliberate projection of `docs/` — when a trap changes, both move together. -- **`mmproj-offload = true` fails silently at startup on a saturated GPU.** CLIP's warmup - compute buffer OOMs but the server keeps running — only image requests error at generation - time. Set `false` on tiers where LLM + KV already saturate VRAM. - -- **All Qwen 3.6, Qwen 3.8 and Bonsai entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** - Required, *not* redundant with `jinja = true` — `chat-template-file` *replaces* - the GGUF-embedded template entirely (`vendor/llama.cpp/common/arg.cpp:3142`, - `params.chat_template = read_file(value)`). The upstream embedded template has - documented issues with tool calls, role handling, `` block rendering, - agentic loops, and llama.cpp KV-prefix cache stability; the vendored template - fixes all of them (full list in `vendor/Qwen-Fixed-Chat-Templates/README.md`). - Since v19 the template is a single unified file, and since v22 it covers Qwen 3.5, - 3.6 *and* 3.8 (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under - `archive/`). The template adds a `<|think_on|>` / `<|think_off|>` toggle, and - defaults `preserve_thinking` to `true` (past `` blocks are kept - chronologically for 100% KV prefix cache stability and agentic reasoning - continuity). To strip past `` blocks instead, set - `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower - KV cache hit rate. v22 also honours `preserve_reasoning`, so `--reasoning-preserve` - (`common/arg.cpp:3677-3689`) works as the CLI equivalent. Path is repo-relative, - so `llama-server` must be launched from the repo root — `read_file()` resolves - against the process CWD, not the INI file's directory. `Qwen3-Coder-Next` entries - deliberately keep their GGUF-embedded template; froggeric's README claims - compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. - -- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry does not.** - v22 added Qwen 3.8's reasoning-effort steering but gates it on nothing — the default - resolves to `xhigh` for *every* model the template serves, injecting a ~45-token - "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. - Qwen 3.6 has no trained notion of the concept, so the entries pin `medium`, the one - level for which v22 emits no instruction text at all. Qwen 3.8 *is* trained on it and - its own template defaults to `xhigh`, so that entry leaves the key unset and inherits - the same default it had before the pin. `--reasoning-effort` writes only a template - kwarg (`common/arg.cpp:3650-3660`), which a request can still override - (`tools/server/server-common.cpp:1296-1303`). Unlike the GGUF-embedded 3.8 template, - v22 never raises on an unknown level — `high` is aliased to `xhigh` and anything else - falls back to it. - -- **`Qwen3.8-27B` pins the template too — v22 removed the reason it used to be the exception.** - Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as - Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, - `eos = 248046`), but it embeds a *different, newer* 8952-byte template, not the - 7764-byte one shared by Qwen3.6-27B and both Bonsai variants. That newer file already - defaults `preserve_thinking` on and adds `reasoning_effort`, which is why the entry - shipped unpinned until v22 supported 3.8. Three defects justify pinning anyway, all - reproducible by rendering the embedded template directly: tool calls whose `arguments` - arrive as a JSON *string* (what most OpenAI-compatible clients send) abort with - `Can only get item pairs from a mapping`; history carrying reasoning inside `content` - rather than `reasoning_content` renders a duplicate blank `\n\n` ahead - of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` - accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, - `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. - v22 handles all three and its `xhigh` instruction text is byte-identical to the - official one, so `reasoning = on` with no `reasoning-effort` key reproduces the - pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is - selected purely on `` + `` block. The pin - additionally brings froggeric's agentic extras (two-tier tool-error escalation, - `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). - -- **v22 fixed two v19 deviations from the official Qwen prompt format, so the bump changed - every pinned entry, not just the new 3.8 one.** v19 serialized `` entries - *unwrapped* (`{"description": ..., "name": ..., "parameters": ...}`); v22 emits the - wrapped OpenAI form (`{"function": {...}, "type": "function"}`), which is what - Qwen3.6-27B's own 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 - was the outlier. v19 also rendered `\n` before assistant content where the - official templates and llama.cpp's own generation prompt use `\n\n` - (`common/chat.cpp:1163`). Both are corrections, but they change prompt bytes, so the - bump invalidates existing KV prefix caches once; `reasoning-effort = medium` suppresses - only the new steering paragraph and does not restore v19 output. - -- **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** - 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself - embeds as `general.sampling.temp`, applied at `common/common.cpp:1264` unless the - preset overrides it. The rest of the sampler block (`top-p 0.95`, `top-k 20`, - `min-p 0.0`, `presence-penalty 0`) is unchanged from the 3.6 entries. Qwen 3.8's - non-thinking mode wants a different set (`temp 0.7`, `top-p 0.8`, - `presence-penalty 1.5`); the preset does not cover it because `reasoning = on`. - -- **Qwen 3.8's MTP head is multi-step trained, so `spec-draft-n-max = 3` is a measured - peak rather than an inherited default.** A day-0 `n_max` sweep of 2/3/4/6 on this model - put the maximum at 3 — acceptance falls monotonically with depth, but through 3 the - extra tokens per iteration win. This overturns the Qwen 3.6 rule of thumb that 2 was - optimal. 3 also happens to be the upstream default (`common/common.h:325`). Note the - cost: `draft-mtp` sets `n_rs_seq = spec-draft-n-max` (`common/common.cpp:1699`), which - multiplies the recurrent-state buffer by `1 + n_max` — ~150 MiB becomes ~600 MiB at 3. - Both `Qwen3.8-27B` and `Qwen3.6-27B` carry `blk.64` (the MTP head) at `Q4_0` in the - local IQ4_XS files; a 4-bit MTP head is reported to collapse acceptance to 0% on this - model family, so check the server's acceptance rate before trusting the speedup — - the fix would be a re-quant keeping `blk.64` at `Q5_K` or above, not a preset change. - -- **`Qwen3.8-27B` is the only 24 GB Qwen entry on a `Q8_0` projector instead of `BF16`.** - 600 MiB rather than 888 MiB of VRAM, and that saving is what keeps `mmproj-offload = true` - affordable at `ctx-size = 262144`: the entry lands at ~19.95 GiB of ~22.6 GiB usable, just - under the Qwen3.6-27B entry's ~20.23 GiB, which leaves room for the CLIP compute buffer. - Spending the saving elsewhere is what breaks it — raising the KV cache to `q5_0` K / `q4_1` V - costs ~0.80 GiB at this context and pushes the total above the 3.6 entry, into the - silent-OOM window described above. Quality is not the tradeoff: Qwen ships this family's - projector as FP16 *and* `Q8_0` officially, and only 83 of the file's 110 weight tensors are - actually 8-bit — every `ffn_down` stays `F16`. - -- **Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from `Qwen/Qwen3.8-27B-FP8`.** BF16 is - this model's native precision and the FP8 repo is a derived, post-training artifact (HF model - tree: base model `Qwen3.8-27B`, "Quantized"), so it is already lossy — its card claims only - "nearly identical" metrics. Quantizing from it would fit the quantizer and the imatrix to - degraded weights. This is the opposite of DeepSeek-V3/V4, which were *trained* in FP8, making - their FP8 checkpoint the original and its dequant to BF16 exact; `convert_hf_to_gguf.py:156` - (`--fp8-as-q8`) exists for that case, not this one. The mmproj is the one exception where the - source does not matter: the FP8 repo leaves the whole vision tower unquantized (0 of 333 - `model.visual.*` tensors carry `weight_scale_inv`), so it is byte-identical either way. The - MTP head is not — `mtp.layers.0`'s attention and MLP projections are FP8 there, so any - re-quant raising `blk.64` above 4-bit must also come from the BF16 repo. - -- **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** - This is Google's fixed official template as aligned by upstream (#21704) — the exact - file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler - (`vendor/llama.cpp/common/chat.cpp:1216`), so parser and template always come from the - same submodule commit (each rebuild resets the submodule to master, mirroring the built - binary). GGUF-embedded templates from conversions predating Google's template fixes - lack the `{#- OpenAI Chat Completions:` marker; llama.cpp then logs "detected an - outdated gemma4 chat template" and rewrites messages via C++ compatibility workarounds - (`common/chat.cpp:2250-2258`) — the pin avoids that path. One file covers the whole - series (12B / 26B-A4B / 31B, incl. `<|image|>`/`<|audio|>` placeholders), and - `reasoning = on` maps to the template's `enable_thinking` kwarg - (`common/arg.cpp:3167-3175`), so no `chat-template-kwargs` are needed. Unlike the Qwen - template, past `<|channel>thought` blocks are *stripped* from history by design — - Gemma 4 is trained that way — so cross-turn KV-prefix invalidation is inherent - (`ctx-checkpoints` mitigates); do not add a preserve-thinking hack. If startup fails - reading the template after a rebuild, check whether upstream moved - `models/templates/` (same failure mode as the `gguf_dump.py` note above). - -- **Both Bonsai entries use the same `chat-template-file` pin as the Qwen 3.6 entries.** - `Ternary-Bonsai-27B` and `Bonsai-27B` ship from separate HF repos but are both Qwen3.6-27B - derivatives: arch `qwen35`, and their tokenizers are byte-identical to stock Qwen3.6-27B - (248320 tokens, same merges, `eos = 248046`) right down to the same 7764-byte embedded template — - which is exactly the upstream template the pin exists to replace. `general.sampling.temp = 1.0` is - embedded in both GGUFs and applied at `common/common.cpp:1264`, so `temp` has to be pinned in the - preset or generation runs at 1.0. The presets use `0.6` to match the sibling Qwen 3.6 entries; - Prism's own card benchmarks at `0.7`. Unlike the DSpark sidecar below, both weight files are - mainline-packed (`Q2_0` at `QK2_0 64`, `Q1_0` at `QK1_0 128`) and load without a tensor-offset - mismatch. - -- **The DSpark drafter shipped beside Ternary Bonsai 27B cannot be enabled on mainline.** - `Ternary-Bonsai-27B-dspark-Q4_1.gguf` has its `token_embd.weight` in `Q2_0` at Prism's - group-128 packing while mainline is group-64 (`QK2_0 64`, `ggml/src/ggml-common.h`), so - `gguf_init_from_reader` rejects the file on a tensor-offset mismatch before any architecture - dispatch. Repacking would not help: `general.architecture = 'dspark'` is unregistered - (`src/llama-arch.cpp:136` has only `dflash`), and mainline's DSpark is DeepSeek-V4 - DFlash + Markov (`src/models/dflash.cpp`, tensors `markov_w1`/`markov_w2`/`conf_proj`, - requiring MLA and sqrtsoftplus MoE scoring), not Prism's 6-layer Qwen3.6-shaped drafter - (`dspark.fc`, `dspark.log_snr_fc*`, `dspark.markov_head_*`). Upstream confirmed on #25707 - that it stays fork-only. The GGUF carries no MTP tensors either, so `draft-mtp` is out and - the entry uses `ngram-mod`. Only the group-64 pack is mainline-loadable — `Q2_0.gguf` and - `PQ2_0.gguf` in the same HF repo are group-128 fork packs. - -- **DeepSeek-V4-Flash-0731 must set `cache-type-k` *and* `cache-type-v` to the same value.** - Arch is `deepseek4` (the HF card's "dflash / 20B" is the DSpark sidecar's metadata, not the - model). `llama-context.cpp:3560-3563` compares the two values and refuses to create the - context — `does not support different K (%s) and V (%s) cache types` — for - `LLM_ARCH_DEEPSEEK4` specifically, because `hparams.is_mla()` is *false* for this arch and the - guard needs the explicit disjunct. So `cache-type-v = q8_0` is load-bearing even though V is - never allocated: DSV4 is K-only everywhere (`dsv4_make_k_only()` at - `llama-kv-cache-dsv4.cpp:831-835` forces `is_mla` true on hparams copies, so - `has_v = !is_mla` at `llama-kv-cache.cpp:229` is false). Copying the Qwen dual-GPU pair - `q5_0` K / `q4_1` V here is startup-fatal, not merely wasteful. `q8_0` also clears - `n_embd_head_k() % 64 == 0` so quantized K gets the Hadamard rotation - (`llama-kv-cache.cpp:319-323`); the lightning-indexer cache is rotated unconditionally for - this arch (`:325-329`). `kv-unified` is silently discarded (`GGML_UNUSED(unified)`, - `dsv4.cpp:1189-1192`), and `cache-type-*-draft` is dead without a draft model. - -- **The DSV4 KV cache is tiny, so context is cheap and quantizing it buys little.** 43 layers - split 2 raw / 21 CSA (ratio 4) / 20 HCA (ratio 128) via `attention.compress_ratios`, all - K-only at `n_embd_k_gqa = 512`; the raw tier is SWA-windowed to - `PAD(min(n_ctx, 128 + n_ubatch), 256)` = 768 cells regardless of `ctx-size`. At 262144 that - is 942 MiB at `q8_0` (1764 MiB at f16) plus a fixed 11.64 MiB of F32 compressor state that no - cache type shrinks. Never set `swa-full`: it collapses the window formula to `n_ctx` - (`llama-kv-cache-iswa.cpp:76-81`), turning a 17 MiB raw cache into ~11 GiB. The server warns - `swa_full is not supported` only *after* the cache is built, so the flag still takes effect. - -- **Leave `fit = on` and never add `n-cpu-moe`/`-ot` to the DeepSeek entry.** Measured UD-Q8_K_XL - composition: 137.06 GiB routed experts (MXFP4, 90.9%), 2.02 GiB shared experts, 11.67 GiB - non-expert — so non-expert + shared is only 13.69 GiB and fits a 24 GB card alongside the KV - with room for a full expert layer or two (3.19 GiB each). `fit` finds that split at sub-layer - granularity (`fit.cpp:399-441`, `:719-769`); forcing `-ncmoe 43` would push all experts to CPU - and strand ~9 GiB of VRAM. Any user `-ot`/`--cpu-moe`/`--n-cpu-moe` aborts fit outright - (`fit.cpp:395-397`), as does setting `n-gpu-layers` to anything but `-1` (`fit.cpp:374-376`) — - which is why the entry keeps `-1` explicitly. Note `--cpu-moe`'s pattern matches only - `_exps`/`_chexps`, so shared experts would stay on GPU either way. - -- **`no-host = true` is mandatory on the DeepSeek entry, and this is the trap that actually stops - it loading.** Unless `no_host` is set, `make_cpu_buft_list()` prepends - `ggml_backend_dev_host_buffer_type()` to the CPU buffer list, so *every* CPU-resident tensor is - allocated in a `CUDA_Host` (page-locked) buffer (`src/llama-model.cpp:896-917`, wired from - `params.no_host` at `common/common.cpp:1611` and `include/llama.h:338`). For this model the - loader then reports one `CUDA_Host model buffer size = 137046.96 MiB` — a 133.8 GiB - `cudaMallocHost` on a 192 GB box. The reservation *succeeds*, so `ggml_cuda_host_malloc`'s - clean-failure fallback to an ordinary CPU buffer never fires; the failure happens later while - the pages are committed during the read and surfaces as `CUDA error: out of memory` inside - `cudaEventSynchronize` at `src/llama-model-loader.cpp:1591`. That makes a host-memory problem - look like a VRAM problem — raising `fit-target` does not help it, and neither does changing - `load-mode`. With `no-host = true` the same config loads in ~144 s at 18650 MiB VRAM and - ~124 GiB of ordinary host RAM. `GGML_CUDA_NO_PINNED=1` is the env-var equivalent. This applies - to any entry that pushes tens of GiB of experts to CPU, not only DeepSeek. The GPU upload - staging buffers are unaffected — the loader asks for those buffer types directly - (`src/llama-model-loader.cpp:1467`) rather than through `cpu_buft_list` — but expert weights - that `op-offload` ships to the GPU for large-batch matmuls now come from pageable memory, which - may cost some prompt-processing throughput. There is no way to keep that and still load. - -- **`fit-target = 3072` on the DeepSeek entry is a WDDM safety margin, not the fix for the load - failure** (that is `no-host` above). `fit` measures rather than guesses — it performs a - `no_alloc` model load plus a real graph reservation (`fit.cpp:56-75`), so its KV figure is - byte-exact (942 MiB at 262144/`q8_0`) and its compute figure is a genuine `ggml_gallocr` - measurement. What it cannot see is the CUDA VMM scratch pool (32 GiB of VA reserved, physical - pages committed on demand, `ggml/src/ggml-cuda/ggml-cuda.cu:536-656`), the lazy cuBLAS - workspace, and CUDA graph instances; none are reported to `memory_breakdown()`. It also takes a - single `cudaMemGetInfo` snapshot at t=0 (`fit.cpp:194`) and carries no WDDM or framebuffer - allowance anywhere. At the default 1024 MiB margin fit keeps blk.0 and blk.1 routed experts on - the GPU (6.375 GiB — every one of the 43 layers carries 3.188 GiB of routed experts, there are - no dense layers) and leaves only 1368 of 23139 usable MiB for those untracked consumers. - `3072` leaves 3497 MiB and costs one extra expert layer on CPU (~2.3% more expert traffic). - Do not raise it to 6144: that collapses `-ngl` to 38 and starts stranding whole layers. - `--fit-target` writes only `params.fit_params_target` and never `mparams`, so unlike - `-ngl`/`-ncmoe`/`-ot` it cannot trip the aborts at `fit.cpp:374-397`. - -- **`cache-ram` is 16384 on the DeepSeek entry, not the 51200 used elsewhere.** `fit` reports - 133.8 GiB of `Host model` weights for this entry (measured at `fit-target = 3072`), so on a - 192 GB box a 50 GiB prompt cache overcommits and pages. A full-context prompt - state at 262144/`q8_0` is 931 MiB (`server-task.cpp:1671-1683` — an entry larger than the - whole limit is silently skipped), so 16 GiB still holds ~17 of them. Context checkpoints are - separate and cheap: 14.5 MiB each and independent of `ctx-size`, because a DSV4 checkpoint - stores only the 128-position SWA window plus the fixed compressor state. - -- **The DeepSeek-V4-Flash entry deliberately does *not* pin `chat-template-file`.** This is the - one exception to the convention above. The GGUF ships Unsloth's fixed template, and both it - and upstream's bundled `models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja` (#26398) - satisfy the detection heuristic at `common/chat.cpp:3170-3179` (`dsml_token` + `DSML` + - `tool_calls`), so both route to the native PEG parser - (`common_chat_params_init_deepseek_v3_2`, `chat.cpp:2097`) and classify as V4 via the - `function_calls`-absent test at `:2105`. Unsloth's additionally restores `reasoning_content` - on tool calls, which the official template drops. Unlike gemma-4 there is no outdated-template - rewrite path for `deepseek4` — detection is all-or-nothing, and a miss degrades to the generic - autoparser rather than being repaired. The entry carries the effort as - `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); `reasoning-effort = high` is - equivalent and simpler — `common/arg.cpp:3650` writes the same template kwarg, and the level - `xhigh` is listed in its help text. Only the *request* field special-cases the literal `"none"` - (`server-common.cpp:1296-1304`). Left unset the template defaults it to `none` and emits no - effort block at all, and `reasoning = off` voids it entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` - (`common.h:631`, despite the help text saying `auto`), and `none` leaks `` into - `content`. - -- **Context shift and cache-reuse are permanently unavailable on `deepseek4`.** - `llama_kv_cache_dsv4::get_can_shift()` returns false (`dsv4.cpp:1394-1398`), so the server - force-disables both with a warning (`server-context.cpp:1268-1278`); slots then stop cleanly - at `STOP_TYPE_LIMIT` instead of shifting. `seq_rm` also refuses partial removal when - `n_rs_seq == 0` (`dsv4.cpp:1427-1429`), which `ngram-mod` does not set, so rollback goes - through checkpoints — correct, but each rejected draft costs a ~14.5 MiB state restore and the - net throughput effect is uncharacterised. The `dspark` sidecar in the same HF repo is a - genuine mainline `dflash` drafter (unlike the Bonsai one above), but is unusable here: its - README requires `--fit off` plus full offload of target *and* drafter (11 GiB drafter + - 13.69 GiB non-expert exceeds 24 GB), `--spec-draft-n-max` is clamped to 5, multi-GPU needs a - rebuild with `GGML_SCHED_MAX_SPLIT_INPUTS=48`, and it carries an open decode-time CUDA abort - after ~2500 tokens (#26554). The regression that broke spec decoding on this arch (#26576, - a 2D `wo_a` in `dflash.cpp` after #26531) is fixed by #26577 at `b10269`. - -- **`ctx-size` above the GGUF's `context_length` is dead VRAM unless `override-kv` lifts it too.** - `llama-context.cpp:131` never clamps `n_ctx`, so the KV cache really is allocated at the - requested size — but the server then caps every slot at `n_ctx_train` - (`tools/server/server-context.cpp:1201-1203`, applied as `slot.n_ctx = n_ctx_slot` at `:1255`) - and rejects any larger request outright at `:3100` / `:3111`. Both Muse Glimmer GGUFs ship - `muse-glimmer.context_length = 131072`, so the 24 GB entry's former `ctx-size = 262144` - allocated 262144 cells while no request could exceed 131072 — ~884 MiB of unreachable VRAM. - `override-kv = muse-glimmer.context_length=int:262144` raises `n_ctx_train` and is the only - lever; Meta documents 131072 as the default and 262144 as the maximum, so this is the - vendor-sanctioned ceiling, not an extrapolation hack. The startup line - `the slot context (...) exceeds the training context of the model (...) - capping` is expected - here (the pool is 524288 for two 262144 slots); what matters is - `initializing, n_slots = 2, n_ctx_slot = 262144`. If that reads 131072 the override did not - take. It also sets `n_ctx_orig_yarn = 262144` (`src/llama-model.cpp:1180`), inert at - `freq_scale 1`. - -- **Muse Glimmer has no architectural positional ceiling, so never add RoPE scaling.** Meta's - `config.json` sets `layer_rope_theta = 0` on all 13 `full_attention` layers (NoPE) and - `sliding_window = 2048` on the other 39; the GGUF's `sliding_window_pattern` confirms the - `[true, true, true, false]` x13 split. RoPE therefore never sees a relative position above 2048 - at any context length — 131072 is a training length, not a limit, which is why Meta writes - "131,072+". Because the GGUF carries no `rope.scaling.*` keys the type defaults to `linear` with - `freq_scale 1` (`src/llama-model.cpp:1187-1189`), so a user `--rope-scale` is *applied*, not - ignored, and would compress local resolution on the SWA layers for no benefit. The - `--rope-scaling yarn` recipes circulating on the HF model page are wrong; only their - `--override-kv` half is load-bearing. - -- **Never set `swa-full` or `context-shift` on a Muse Glimmer entry.** `swa-full` collapses the - 4608-cell SWA cache to the full `n_ctx` by the mechanism documented for DeepSeek above — a ~114x - blowup at `ctx-size = 524288`. `context-shift` is the more dangerous one, and it is the exact - inverse of the `deepseek4` case: `llama_hparams::has_rope()` knows only about `router_layer` - (`src/llama-hparams.cpp:287-295`), so a K-shift rotates 128 dims on all 52 layers including the - 13 NoPE ones, while `get_can_shift()` returns true. Silent corruption rather than a refusal. It - is dormant only because `ctx_shift` defaults false (`common/common.h:561`) and loading an mmproj - force-disables it (`server-context.cpp:1163-1171`); dropping the mmproj would make it reachable. - -- **`spec-draft-n-max = 15` is `block_size - 1`, and the absent `chat-template-file` and - `reasoning` keys are both deliberate.** The drafter carries `dflash.block_size = 16` and spends - block position 0 on the committed anchor, so 15 is the maximum legal draft - (`common/speculative.cpp:970-978`, no clamp and no warning); the upstream default is 3 - (`common/common.h:325`), so leaving it unset discards 80% of a block that is decoded in one - forward pass regardless. Judge it by `mean len` in the slot timings - (`1 + n_accepted / n_verif_steps`, `server-context.cpp:620`), never by `draft acceptance` — that - ratio is a percentage of *drafted* tokens and is meaningless for block diffusion, where - unaccepted drafts cost nothing. Measured ~3.4 tokens per target pass at short context, ~3.0 at - 4.3k. No template may be pinned: `common/chat.cpp:3276` selects the native Muse Glimmer handler - on `` + `<|eom|>` in the template source, and there is no bundled file to - pin. `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads - `enable_thinking`; reasoning here is the structural ` to=self<|message|>...<|eom|>` channel, - gated on `reasoning_format` (`chat.cpp:3141`). The template already defaults to - `Reasoning strength: high` (`models/templates/muse-glimmer.jinja:84`), which matches Meta's - recommendation, and `--reasoning-effort` reaches it only through the - `reasoning_effort` -> `reasoning_strength` alias at `common/jinja/caps.cpp:29-33`. - `--reasoning-budget` is inert: the handler sets no `thinking_end_tags` - (`server-common.cpp:1343`). - -- **Compute buffers dominate this entry's headroom and cannot be derived from KV arithmetic.** - Measured on a 24463 MiB card at `ctx-size = 524288` / `parallel = 2` / `q5_0` K + `q4_1` V: - 13488.92 MiB target weights, 1371.40 draft, 1956.60 CLIP, 2271.12 KV (2184.00 non-SWA + - 57.59 SWA + 29.53 draft) and **2655.59 MiB of compute buffers** (1135.67 target + 803.03 - spec-context + 407.62 draft + 309.27 CLIP) — 21743.63 MiB total, leaving ~1.0 GiB free. The - compute term is larger than the whole KV cache and grew 1070.67 -> 1135.67 MiB once a real - request arrived, so budget it explicitly instead of sizing `ctx-size` off KV alone. Note the - SWA cache scales with `parallel`, not `ctx-size` - (`PAD(min(n_ctx, n_swa * n_seq_max + n_ubatch), 256)` = 4608 cells), so raising `parallel` - is cheap while raising `ctx-size` is not. `mmproj-offload = true` survives this margin (CLIP - warmup reserves 309.27 MiB and real images decode), but it is the first thing to disable if the - margin shrinks — see the silent-OOM note above. Context checkpoints are host-side, ~38.8 MiB - each at `max = 32`. - -**ngram-mod speculative decoding** (`--spec-type ngram-mod`): model-agnostic, works on any model. -- All models: `spec-ngram-mod-n-match = 24`, `spec-ngram-mod-n-min = 48`, `spec-ngram-mod-n-max = 64` - (matches the struct defaults in `common/common.h:329-337` and what `--spec-default` produces - at `common/arg.cpp:4065-4074`; ggerganov confirmed post-merge in PR #19164 that the min/max - "likely don't need to be changed from the recommended values"; MoEs require long drafts and - dense models tolerate them without noticeable cost). Flags were renamed from - `--draft-min`/`--draft-max`/`--spec-ngram-size-n` in upstream PR #22397; the old names now - error at startup. -- `n_match < 16` logs a "too small — poor quality is possible" warning at - `vendor/llama.cpp/common/speculative.cpp:1031-1034`; parser accepts `1..1024` - (`common/arg.cpp:3606-3615`), so 16 is the lowest non-warning value, not a hard floor. - Min/max parsers accept `0..1024` (`common/arg.cpp:3587-3605`). -- Memory overhead: ~16 MiB **total**, shared across all server slots - (single `common_ngram_mod` instance allocated at `common/speculative.cpp:1026`). -- Pool auto-resets on `begin()` if occupancy > 25 %, and after 3 consecutive rounds with - acceptance < 50 % (`common/speculative.cpp:720-728`, `:790-806`). Smaller `n_match` makes - these resets fire more often and wipes ngrams learned from the current prompt — another - reason to stay at `n_match ≥ 24`. +- Never pair `direct-io` with `no-mmap`; use `load-mode = dio`. `docs/presets.md` -> *load-mode* +- Never set `mmproj-offload = true` on a tier where LLM + KV already saturate VRAM. `docs/presets.md` -> *mmproj-offload* +- Never set `swa-full` on a DeepSeek-V4-Flash or Muse Glimmer entry. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Muse Glimmer* +- Never set `context-shift` on a Muse Glimmer entry; it is silent corruption, not a refusal. `docs/model_tuning.md` -> *Muse Glimmer* +- Never add RoPE scaling to a Muse Glimmer entry. `docs/model_tuning.md` -> *Muse Glimmer* +- `no-host = true` is mandatory on the DeepSeek entry; without it the load fails as a misleading CUDA OOM. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* +- Keep `fit = on` on the DeepSeek entry; never add `n-cpu-moe`/`-ot`, and never set `n-gpu-layers` to anything but `-1`. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* +- `cache-type-k` and `cache-type-v` must be identical on `deepseek4`; differing values are startup-fatal. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* +- Never set `image-min-tokens` on a gemma-4 entry; it is a `qwen3vl_merger` key only. `docs/model_tuning.md` -> *Qwen 3.6 and 3.8* +- Never drop a `chat-template-file` pin; it replaces the GGUF-embedded template and is not redundant with `jinja = true`. `docs/model_tuning.md` +- Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from the derived `-FP8` repo. `docs/model_tuning.md` -> *Qwen 3.6 and 3.8* ## Changelog style - One bullet = one physical line. Never insert manual line-breaks; let the editor soft-wrap. - Format: `- [Component] ` (Added / Changed / Fixed / Removed). -- No rationale, no file paths, no line numbers, no explanatory prose. Rationale lives in AGENTS.md "Non-obvious behavior" or in the commit message. +- No rationale, no file paths, no line numbers, no explanatory prose. Rationale lives in AGENTS.md "Non-obvious behavior", the matching `docs/` file, or the commit message. - PR refs as bare `#NNNNN`, at most once per release. - Canonical examples: [1.21.0] – [1.27.0] in CHANGELOG.md. ## Scratch Files Non-committed agent artifacts (diffs, trace outputs, generated reports, experimental scripts) go under `.tmp/sessions//` at the repo root; `.tmp/` is gitignored. `` is `SESSION_ID` when the platform injects it, otherwise a minted `YYYYMMDD-HHMMSS-`. Never write scratch files to `.claude/`, the repo root, or `vendor/`. + +## Reference + +Deep reference documentation lives under `docs/` and is **read on demand**, not loaded into context. Consult the relevant file when a task touches its area: + +| Document | When to read | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs/presets.md` | Cross-model INI rules: device pinning and multi-GPU, `load-mode`, `mmproj-offload`, context size and `override-kv`, ngram-mod speculative decoding. **Read before editing any file under `presets/`.** | +| `docs/model_tuning.md` | Per-family rationale and measured VRAM/throughput numbers for Qwen 3.6 and 3.8, gemma-4, Bonsai and DSpark, DeepSeek-V4-Flash, and Muse Glimmer. **Read before adding, retuning or removing a model entry.** | diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e02244..f331166 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document the measured Muse-Glimmer-30B 24 GB compute buffer and VRAM breakdown ### Changed +- [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md +- [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md - [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 diff --git a/docs/model_tuning.md b/docs/model_tuning.md new file mode 100644 index 0000000..5c3564b --- /dev/null +++ b/docs/model_tuning.md @@ -0,0 +1,340 @@ +# Model Tuning Reference + +Per-model-family rationale behind the entries in `presets/*.ini`, with the measured +numbers each decision rests on. Not auto-loaded into the agent context; read the +relevant section on demand. Cross-model rules are in `docs/presets.md`. + +## Qwen 3.6 and 3.8 + +- **Qwen-VL entries pin `image-min-tokens = 1024`.** `clip.cpp:1500` sets the per-image token limits + to `(8, 4096)`, so the default *minimum* is 8 tokens — 8192 px at `merge = 2` / `patch = 16` — and + `clip.cpp:1502-1506` warns on every load because upstream needs >= 1024 tokens (1024x1024 px) for + grounding (#16842). The key raises a floor only: images already above 1024 tokens are unchanged, + smaller ones get upscaled, which costs context and CLIP time on the CPU because the 16 GB tier + sets `no-mmproj-offload = true`; the 24 GB `Qwen3.8-27B` entry offloads CLIP to the GPU and + pays it there instead. Applies to the `qwen3vl_merger` entries (Qwen3.6, Qwen3.8 and + Ternary-Bonsai); gemma-4 uses a different projector and must not get this key. + +- **All Qwen 3.6, Qwen 3.8 and Bonsai entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** + Required, *not* redundant with `jinja = true` — `chat-template-file` *replaces* + the GGUF-embedded template entirely (`vendor/llama.cpp/common/arg.cpp:3142`, + `params.chat_template = read_file(value)`). The upstream embedded template has + documented issues with tool calls, role handling, `` block rendering, + agentic loops, and llama.cpp KV-prefix cache stability; the vendored template + fixes all of them (full list in `vendor/Qwen-Fixed-Chat-Templates/README.md`). + Since v19 the template is a single unified file, and since v22 it covers Qwen 3.5, + 3.6 *and* 3.8 (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under + `archive/`). The template adds a `<|think_on|>` / `<|think_off|>` toggle, and + defaults `preserve_thinking` to `true` (past `` blocks are kept + chronologically for 100% KV prefix cache stability and agentic reasoning + continuity). To strip past `` blocks instead, set + `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower + KV cache hit rate. v22 also honours `preserve_reasoning`, so `--reasoning-preserve` + (`common/arg.cpp:3677-3689`) works as the CLI equivalent. Path is repo-relative, + so `llama-server` must be launched from the repo root — `read_file()` resolves + against the process CWD, not the INI file's directory. `Qwen3-Coder-Next` entries + deliberately keep their GGUF-embedded template; froggeric's README claims + compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. + +- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry does not.** + v22 added Qwen 3.8's reasoning-effort steering but gates it on nothing — the default + resolves to `xhigh` for *every* model the template serves, injecting a ~45-token + "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. + Qwen 3.6 has no trained notion of the concept, so the entries pin `medium`, the one + level for which v22 emits no instruction text at all. Qwen 3.8 *is* trained on it and + its own template defaults to `xhigh`, so that entry leaves the key unset and inherits + the same default it had before the pin. `--reasoning-effort` writes only a template + kwarg (`common/arg.cpp:3650-3660`), which a request can still override + (`tools/server/server-common.cpp:1296-1303`). Unlike the GGUF-embedded 3.8 template, + v22 never raises on an unknown level — `high` is aliased to `xhigh` and anything else + falls back to it. + +- **`Qwen3.8-27B` pins the template too — v22 removed the reason it used to be the exception.** + Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as + Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, + `eos = 248046`), but it embeds a *different, newer* 8952-byte template, not the + 7764-byte one shared by Qwen3.6-27B and both Bonsai variants. That newer file already + defaults `preserve_thinking` on and adds `reasoning_effort`, which is why the entry + shipped unpinned until v22 supported 3.8. Three defects justify pinning anyway, all + reproducible by rendering the embedded template directly: tool calls whose `arguments` + arrive as a JSON *string* (what most OpenAI-compatible clients send) abort with + `Can only get item pairs from a mapping`; history carrying reasoning inside `content` + rather than `reasoning_content` renders a duplicate blank `\n\n` ahead + of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` + accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, + `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. + v22 handles all three and its `xhigh` instruction text is byte-identical to the + official one, so `reasoning = on` with no `reasoning-effort` key reproduces the + pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is + selected purely on `` + `` block. The pin + additionally brings froggeric's agentic extras (two-tier tool-error escalation, + `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). + +- **v22 fixed two v19 deviations from the official Qwen prompt format, so the bump changed + every pinned entry, not just the new 3.8 one.** v19 serialized `` entries + *unwrapped* (`{"description": ..., "name": ..., "parameters": ...}`); v22 emits the + wrapped OpenAI form (`{"function": {...}, "type": "function"}`), which is what + Qwen3.6-27B's own 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 + was the outlier. v19 also rendered `\n` before assistant content where the + official templates and llama.cpp's own generation prompt use `\n\n` + (`common/chat.cpp:1163`). Both are corrections, but they change prompt bytes, so the + bump invalidates existing KV prefix caches once; `reasoning-effort = medium` suppresses + only the new steering paragraph and does not restore v19 output. + +- **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** + 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself + embeds as `general.sampling.temp`, applied at `common/common.cpp:1264` unless the + preset overrides it. The rest of the sampler block (`top-p 0.95`, `top-k 20`, + `min-p 0.0`, `presence-penalty 0`) is unchanged from the 3.6 entries. Qwen 3.8's + non-thinking mode wants a different set (`temp 0.7`, `top-p 0.8`, + `presence-penalty 1.5`); the preset does not cover it because `reasoning = on`. + +- **Qwen 3.8's MTP head is multi-step trained, so `spec-draft-n-max = 3` is a measured + peak rather than an inherited default.** A day-0 `n_max` sweep of 2/3/4/6 on this model + put the maximum at 3 — acceptance falls monotonically with depth, but through 3 the + extra tokens per iteration win. This overturns the Qwen 3.6 rule of thumb that 2 was + optimal. 3 also happens to be the upstream default (`common/common.h:325`). Note the + cost: `draft-mtp` sets `n_rs_seq = spec-draft-n-max` (`common/common.cpp:1699`), which + multiplies the recurrent-state buffer by `1 + n_max` — ~150 MiB becomes ~600 MiB at 3. + Both `Qwen3.8-27B` and `Qwen3.6-27B` carry `blk.64` (the MTP head) at `Q4_0` in the + local IQ4_XS files; a 4-bit MTP head is reported to collapse acceptance to 0% on this + model family, so check the server's acceptance rate before trusting the speedup — + the fix would be a re-quant keeping `blk.64` at `Q5_K` or above, not a preset change. + +- **`Qwen3.8-27B` is the only 24 GB Qwen entry on a `Q8_0` projector instead of `BF16`.** + 600 MiB rather than 888 MiB of VRAM, and that saving is what keeps `mmproj-offload = true` + affordable at `ctx-size = 262144`: the entry lands at ~19.95 GiB of ~22.6 GiB usable, just + under the Qwen3.6-27B entry's ~20.23 GiB, which leaves room for the CLIP compute buffer. + Spending the saving elsewhere is what breaks it — raising the KV cache to `q5_0` K / `q4_1` V + costs ~0.80 GiB at this context and pushes the total above the 3.6 entry, into the + silent-OOM window described in `docs/presets.md` -> *mmproj-offload*. Quality is not the tradeoff: Qwen ships this family's + projector as FP16 *and* `Q8_0` officially, and only 83 of the file's 110 weight tensors are + actually 8-bit — every `ffn_down` stays `F16`. + +- **Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from `Qwen/Qwen3.8-27B-FP8`.** BF16 is + this model's native precision and the FP8 repo is a derived, post-training artifact (HF model + tree: base model `Qwen3.8-27B`, "Quantized"), so it is already lossy — its card claims only + "nearly identical" metrics. Quantizing from it would fit the quantizer and the imatrix to + degraded weights. This is the opposite of DeepSeek-V3/V4, which were *trained* in FP8, making + their FP8 checkpoint the original and its dequant to BF16 exact; `convert_hf_to_gguf.py:156` + (`--fp8-as-q8`) exists for that case, not this one. The mmproj is the one exception where the + source does not matter: the FP8 repo leaves the whole vision tower unquantized (0 of 333 + `model.visual.*` tensors carry `weight_scale_inv`), so it is byte-identical either way. The + MTP head is not — `mtp.layers.0`'s attention and MLP projections are FP8 there, so any + re-quant raising `blk.64` above 4-bit must also come from the BF16 repo. + +## gemma-4 + +- **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** + This is Google's fixed official template as aligned by upstream (#21704) — the exact + file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler + (`vendor/llama.cpp/common/chat.cpp:1216`), so parser and template always come from the + same submodule commit (each rebuild resets the submodule to master, mirroring the built + binary). GGUF-embedded templates from conversions predating Google's template fixes + lack the `{#- OpenAI Chat Completions:` marker; llama.cpp then logs "detected an + outdated gemma4 chat template" and rewrites messages via C++ compatibility workarounds + (`common/chat.cpp:2250-2258`) — the pin avoids that path. One file covers the whole + series (12B / 26B-A4B / 31B, incl. `<|image|>`/`<|audio|>` placeholders), and + `reasoning = on` maps to the template's `enable_thinking` kwarg + (`common/arg.cpp:3167-3175`), so no `chat-template-kwargs` are needed. Unlike the Qwen + template, past `<|channel>thought` blocks are *stripped* from history by design — + Gemma 4 is trained that way — so cross-turn KV-prefix invalidation is inherent + (`ctx-checkpoints` mitigates); do not add a preserve-thinking hack. If startup fails + reading the template after a rebuild, check whether upstream moved + `models/templates/` (same failure mode as the `gguf_dump.py` note in AGENTS.md + "Non-obvious behavior"). + +## Bonsai and DSpark + +- **Both Bonsai entries use the same `chat-template-file` pin as the Qwen 3.6 entries.** + `Ternary-Bonsai-27B` and `Bonsai-27B` ship from separate HF repos but are both Qwen3.6-27B + derivatives: arch `qwen35`, and their tokenizers are byte-identical to stock Qwen3.6-27B + (248320 tokens, same merges, `eos = 248046`) right down to the same 7764-byte embedded template — + which is exactly the upstream template the pin exists to replace. `general.sampling.temp = 1.0` is + embedded in both GGUFs and applied at `common/common.cpp:1264`, so `temp` has to be pinned in the + preset or generation runs at 1.0. The presets use `0.6` to match the sibling Qwen 3.6 entries; + Prism's own card benchmarks at `0.7`. Unlike the DSpark sidecar below, both weight files are + mainline-packed (`Q2_0` at `QK2_0 64`, `Q1_0` at `QK1_0 128`) and load without a tensor-offset + mismatch. + +- **The DSpark drafter shipped beside Ternary Bonsai 27B cannot be enabled on mainline.** + `Ternary-Bonsai-27B-dspark-Q4_1.gguf` has its `token_embd.weight` in `Q2_0` at Prism's + group-128 packing while mainline is group-64 (`QK2_0 64`, `ggml/src/ggml-common.h`), so + `gguf_init_from_reader` rejects the file on a tensor-offset mismatch before any architecture + dispatch. Repacking would not help: `general.architecture = 'dspark'` is unregistered + (`src/llama-arch.cpp:136` has only `dflash`), and mainline's DSpark is DeepSeek-V4 + DFlash + Markov (`src/models/dflash.cpp`, tensors `markov_w1`/`markov_w2`/`conf_proj`, + requiring MLA and sqrtsoftplus MoE scoring), not Prism's 6-layer Qwen3.6-shaped drafter + (`dspark.fc`, `dspark.log_snr_fc*`, `dspark.markov_head_*`). Upstream confirmed on #25707 + that it stays fork-only. The GGUF carries no MTP tensors either, so `draft-mtp` is out and + the entry uses `ngram-mod`. Only the group-64 pack is mainline-loadable — `Q2_0.gguf` and + `PQ2_0.gguf` in the same HF repo are group-128 fork packs. + +## DeepSeek-V4-Flash + +- **DeepSeek-V4-Flash-0731 must set `cache-type-k` *and* `cache-type-v` to the same value.** + Arch is `deepseek4` (the HF card's "dflash / 20B" is the DSpark sidecar's metadata, not the + model). `llama-context.cpp:3560-3563` compares the two values and refuses to create the + context — `does not support different K (%s) and V (%s) cache types` — for + `LLM_ARCH_DEEPSEEK4` specifically, because `hparams.is_mla()` is *false* for this arch and the + guard needs the explicit disjunct. So `cache-type-v = q8_0` is load-bearing even though V is + never allocated: DSV4 is K-only everywhere (`dsv4_make_k_only()` at + `llama-kv-cache-dsv4.cpp:831-835` forces `is_mla` true on hparams copies, so + `has_v = !is_mla` at `llama-kv-cache.cpp:229` is false). Copying the Qwen dual-GPU pair + `q5_0` K / `q4_1` V here is startup-fatal, not merely wasteful. `q8_0` also clears + `n_embd_head_k() % 64 == 0` so quantized K gets the Hadamard rotation + (`llama-kv-cache.cpp:319-323`); the lightning-indexer cache is rotated unconditionally for + this arch (`:325-329`). `kv-unified` is silently discarded (`GGML_UNUSED(unified)`, + `dsv4.cpp:1189-1192`), and `cache-type-*-draft` is dead without a draft model. + +- **The DSV4 KV cache is tiny, so context is cheap and quantizing it buys little.** 43 layers + split 2 raw / 21 CSA (ratio 4) / 20 HCA (ratio 128) via `attention.compress_ratios`, all + K-only at `n_embd_k_gqa = 512`; the raw tier is SWA-windowed to + `PAD(min(n_ctx, 128 + n_ubatch), 256)` = 768 cells regardless of `ctx-size`. At 262144 that + is 942 MiB at `q8_0` (1764 MiB at f16) plus a fixed 11.64 MiB of F32 compressor state that no + cache type shrinks. Never set `swa-full`: it collapses the window formula to `n_ctx` + (`llama-kv-cache-iswa.cpp:76-81`), turning a 17 MiB raw cache into ~11 GiB. The server warns + `swa_full is not supported` only *after* the cache is built, so the flag still takes effect. + +- **Leave `fit = on` and never add `n-cpu-moe`/`-ot` to the DeepSeek entry.** Measured UD-Q8_K_XL + composition: 137.06 GiB routed experts (MXFP4, 90.9%), 2.02 GiB shared experts, 11.67 GiB + non-expert — so non-expert + shared is only 13.69 GiB and fits a 24 GB card alongside the KV + with room for a full expert layer or two (3.19 GiB each). `fit` finds that split at sub-layer + granularity (`fit.cpp:399-441`, `:719-769`); forcing `-ncmoe 43` would push all experts to CPU + and strand ~9 GiB of VRAM. Any user `-ot`/`--cpu-moe`/`--n-cpu-moe` aborts fit outright + (`fit.cpp:395-397`), as does setting `n-gpu-layers` to anything but `-1` (`fit.cpp:374-376`) — + which is why the entry keeps `-1` explicitly. Note `--cpu-moe`'s pattern matches only + `_exps`/`_chexps`, so shared experts would stay on GPU either way. + +- **`no-host = true` is mandatory on the DeepSeek entry, and this is the trap that actually stops + it loading.** Unless `no_host` is set, `make_cpu_buft_list()` prepends + `ggml_backend_dev_host_buffer_type()` to the CPU buffer list, so *every* CPU-resident tensor is + allocated in a `CUDA_Host` (page-locked) buffer (`src/llama-model.cpp:896-917`, wired from + `params.no_host` at `common/common.cpp:1611` and `include/llama.h:338`). For this model the + loader then reports one `CUDA_Host model buffer size = 137046.96 MiB` — a 133.8 GiB + `cudaMallocHost` on a 192 GB box. The reservation *succeeds*, so `ggml_cuda_host_malloc`'s + clean-failure fallback to an ordinary CPU buffer never fires; the failure happens later while + the pages are committed during the read and surfaces as `CUDA error: out of memory` inside + `cudaEventSynchronize` at `src/llama-model-loader.cpp:1591`. That makes a host-memory problem + look like a VRAM problem — raising `fit-target` does not help it, and neither does changing + `load-mode`. With `no-host = true` the same config loads in ~144 s at 18650 MiB VRAM and + ~124 GiB of ordinary host RAM. `GGML_CUDA_NO_PINNED=1` is the env-var equivalent. This applies + to any entry that pushes tens of GiB of experts to CPU, not only DeepSeek. The GPU upload + staging buffers are unaffected — the loader asks for those buffer types directly + (`src/llama-model-loader.cpp:1467`) rather than through `cpu_buft_list` — but expert weights + that `op-offload` ships to the GPU for large-batch matmuls now come from pageable memory, which + may cost some prompt-processing throughput. There is no way to keep that and still load. + +- **`fit-target = 3072` on the DeepSeek entry is a WDDM safety margin, not the fix for the load + failure** (that is `no-host` above). `fit` measures rather than guesses — it performs a + `no_alloc` model load plus a real graph reservation (`fit.cpp:56-75`), so its KV figure is + byte-exact (942 MiB at 262144/`q8_0`) and its compute figure is a genuine `ggml_gallocr` + measurement. What it cannot see is the CUDA VMM scratch pool (32 GiB of VA reserved, physical + pages committed on demand, `ggml/src/ggml-cuda/ggml-cuda.cu:536-656`), the lazy cuBLAS + workspace, and CUDA graph instances; none are reported to `memory_breakdown()`. It also takes a + single `cudaMemGetInfo` snapshot at t=0 (`fit.cpp:194`) and carries no WDDM or framebuffer + allowance anywhere. At the default 1024 MiB margin fit keeps blk.0 and blk.1 routed experts on + the GPU (6.375 GiB — every one of the 43 layers carries 3.188 GiB of routed experts, there are + no dense layers) and leaves only 1368 of 23139 usable MiB for those untracked consumers. + `3072` leaves 3497 MiB and costs one extra expert layer on CPU (~2.3% more expert traffic). + Do not raise it to 6144: that collapses `-ngl` to 38 and starts stranding whole layers. + `--fit-target` writes only `params.fit_params_target` and never `mparams`, so unlike + `-ngl`/`-ncmoe`/`-ot` it cannot trip the aborts at `fit.cpp:374-397`. + +- **`cache-ram` is 16384 on the DeepSeek entry, not the 51200 used elsewhere.** `fit` reports + 133.8 GiB of `Host model` weights for this entry (measured at `fit-target = 3072`), so on a + 192 GB box a 50 GiB prompt cache overcommits and pages. A full-context prompt + state at 262144/`q8_0` is 931 MiB (`server-task.cpp:1671-1683` — an entry larger than the + whole limit is silently skipped), so 16 GiB still holds ~17 of them. Context checkpoints are + separate and cheap: 14.5 MiB each and independent of `ctx-size`, because a DSV4 checkpoint + stores only the 128-position SWA window plus the fixed compressor state. + +- **The DeepSeek-V4-Flash entry deliberately does *not* pin `chat-template-file`.** This is the + one exception to the convention above. The GGUF ships Unsloth's fixed template, and both it + and upstream's bundled `models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja` (#26398) + satisfy the detection heuristic at `common/chat.cpp:3170-3179` (`dsml_token` + `DSML` + + `tool_calls`), so both route to the native PEG parser + (`common_chat_params_init_deepseek_v3_2`, `chat.cpp:2097`) and classify as V4 via the + `function_calls`-absent test at `:2105`. Unsloth's additionally restores `reasoning_content` + on tool calls, which the official template drops. Unlike gemma-4 there is no outdated-template + rewrite path for `deepseek4` — detection is all-or-nothing, and a miss degrades to the generic + autoparser rather than being repaired. The entry carries the effort as + `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); `reasoning-effort = high` is + equivalent and simpler — `common/arg.cpp:3650` writes the same template kwarg, and the level + `xhigh` is listed in its help text. Only the *request* field special-cases the literal `"none"` + (`server-common.cpp:1296-1304`). Left unset the template defaults it to `none` and emits no + effort block at all, and `reasoning = off` voids it entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` + (`common.h:631`, despite the help text saying `auto`), and `none` leaks `` into + `content`. + +- **Context shift and cache-reuse are permanently unavailable on `deepseek4`.** + `llama_kv_cache_dsv4::get_can_shift()` returns false (`dsv4.cpp:1394-1398`), so the server + force-disables both with a warning (`server-context.cpp:1268-1278`); slots then stop cleanly + at `STOP_TYPE_LIMIT` instead of shifting. `seq_rm` also refuses partial removal when + `n_rs_seq == 0` (`dsv4.cpp:1427-1429`), which `ngram-mod` does not set, so rollback goes + through checkpoints — correct, but each rejected draft costs a ~14.5 MiB state restore and the + net throughput effect is uncharacterised. The `dspark` sidecar in the same HF repo is a + genuine mainline `dflash` drafter (unlike the Bonsai one above), but is unusable here: its + README requires `--fit off` plus full offload of target *and* drafter (11 GiB drafter + + 13.69 GiB non-expert exceeds 24 GB), `--spec-draft-n-max` is clamped to 5, multi-GPU needs a + rebuild with `GGML_SCHED_MAX_SPLIT_INPUTS=48`, and it carries an open decode-time CUDA abort + after ~2500 tokens (#26554). The regression that broke spec decoding on this arch (#26576, + a 2D `wo_a` in `dflash.cpp` after #26531) is fixed by #26577 at `b10269`. + +## Muse Glimmer + +- **Muse Glimmer has no architectural positional ceiling, so never add RoPE scaling.** Meta's + `config.json` sets `layer_rope_theta = 0` on all 13 `full_attention` layers (NoPE) and + `sliding_window = 2048` on the other 39; the GGUF's `sliding_window_pattern` confirms the + `[true, true, true, false]` x13 split. RoPE therefore never sees a relative position above 2048 + at any context length — 131072 is a training length, not a limit, which is why Meta writes + "131,072+". Because the GGUF carries no `rope.scaling.*` keys the type defaults to `linear` with + `freq_scale 1` (`src/llama-model.cpp:1187-1189`), so a user `--rope-scale` is *applied*, not + ignored, and would compress local resolution on the SWA layers for no benefit. The + `--rope-scaling yarn` recipes circulating on the HF model page are wrong; only their + `--override-kv` half is load-bearing. + +- **Never set `swa-full` or `context-shift` on a Muse Glimmer entry.** `swa-full` collapses the + 4608-cell SWA cache to the full `n_ctx` by the mechanism documented for DeepSeek above — a ~114x + blowup at `ctx-size = 524288`. `context-shift` is the more dangerous one, and it is the exact + inverse of the `deepseek4` case: `llama_hparams::has_rope()` knows only about `router_layer` + (`src/llama-hparams.cpp:287-295`), so a K-shift rotates 128 dims on all 52 layers including the + 13 NoPE ones, while `get_can_shift()` returns true. Silent corruption rather than a refusal. It + is dormant only because `ctx_shift` defaults false (`common/common.h:561`) and loading an mmproj + force-disables it (`server-context.cpp:1163-1171`); dropping the mmproj would make it reachable. + +- **`spec-draft-n-max = 15` is `block_size - 1`, and the absent `chat-template-file` and + `reasoning` keys are both deliberate.** The drafter carries `dflash.block_size = 16` and spends + block position 0 on the committed anchor, so 15 is the maximum legal draft + (`common/speculative.cpp:970-978`, no clamp and no warning); the upstream default is 3 + (`common/common.h:325`), so leaving it unset discards 80% of a block that is decoded in one + forward pass regardless. Judge it by `mean len` in the slot timings + (`1 + n_accepted / n_verif_steps`, `server-context.cpp:620`), never by `draft acceptance` — that + ratio is a percentage of *drafted* tokens and is meaningless for block diffusion, where + unaccepted drafts cost nothing. Measured ~3.4 tokens per target pass at short context, ~3.0 at + 4.3k. No template may be pinned: `common/chat.cpp:3276` selects the native Muse Glimmer handler + on `` + `<|eom|>` in the template source, and there is no bundled file to + pin. `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads + `enable_thinking`; reasoning here is the structural ` to=self<|message|>...<|eom|>` channel, + gated on `reasoning_format` (`chat.cpp:3141`). The template already defaults to + `Reasoning strength: high` (`models/templates/muse-glimmer.jinja:84`), which matches Meta's + recommendation, and `--reasoning-effort` reaches it only through the + `reasoning_effort` -> `reasoning_strength` alias at `common/jinja/caps.cpp:29-33`. + `--reasoning-budget` is inert: the handler sets no `thinking_end_tags` + (`server-common.cpp:1343`). + +- **Compute buffers dominate this entry's headroom and cannot be derived from KV arithmetic.** + Measured on a 24463 MiB card at `ctx-size = 524288` / `parallel = 2` / `q5_0` K + `q4_1` V: + 13488.92 MiB target weights, 1371.40 draft, 1956.60 CLIP, 2271.12 KV (2184.00 non-SWA + + 57.59 SWA + 29.53 draft) and **2655.59 MiB of compute buffers** (1135.67 target + 803.03 + spec-context + 407.62 draft + 309.27 CLIP) — 21743.63 MiB total, leaving ~1.0 GiB free. The + compute term is larger than the whole KV cache and grew 1070.67 -> 1135.67 MiB once a real + request arrived, so budget it explicitly instead of sizing `ctx-size` off KV alone. Note the + SWA cache scales with `parallel`, not `ctx-size` + (`PAD(min(n_ctx, n_swa * n_seq_max + n_ubatch), 256)` = 4608 cells), so raising `parallel` + is cheap while raising `ctx-size` is not. `mmproj-offload = true` survives this margin (CLIP + warmup reserves 309.27 MiB and real images decode), but it is the first thing to disable if the + margin shrinks — see `docs/presets.md` -> *mmproj-offload*. Context checkpoints are host-side, ~38.8 MiB + each at `max = 32`. diff --git a/docs/presets.md b/docs/presets.md new file mode 100644 index 0000000..2866cf3 --- /dev/null +++ b/docs/presets.md @@ -0,0 +1,84 @@ +# Presets Reference + +Cross-model rules for editing `presets/*.ini`. Not auto-loaded into the agent +context; read it on demand. `presets/README.md` is the user-facing quick-start; +this file is for editing. Per-model rationale lives in `docs/model_tuning.md`. + +## Editing presets + +VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, +`presets/models_16GB_8GB_VRAM.ini` (dual-GPU). + +## Device pinning and multi-GPU + +- **Only `models_16GB_8GB_VRAM.ini` pins devices.** It sets `split-mode`, `main-gpu`, and + `tensor-split` in its `[*]` section; the other two tiers set none of the three, so on a host with + more than one CUDA device llama.cpp's default `split-mode = layer` spreads every entry across + *all* visible GPUs — the tier name is then a floor, not a cap. Pin with `CUDA_VISIBLE_DEVICES` + before launch, not `--device`: in router mode each child's argv is rebuilt from the preset + (`inst.meta.update_args`), so a parent `--device` never reaches the child, while the environment + is copied into every child (`tools/server/server-models.cpp:802`). Pinning is a throughput + decision, not only a memory one — `Ternary-Bonsai-27B-Q2_g64.gguf` measured 56.5 t/s tg and + 1418 t/s pp on one 16 GB card versus 36.5 t/s and 998 t/s spread over a 16 GB plus an 8 GB card. + `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and + therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. + +## load-mode + +- **All entries use `load-mode = dio`; never pair `direct-io` with `no-mmap` again.** Both spellings + are deprecated, and they write the *same* mutually exclusive enum — `--no-mmap` sets + `LLAMA_LOAD_MODE_NONE` (`common/arg.cpp:2594`) while `--direct-io` sets + `LLAMA_LOAD_MODE_DIRECT_IO` (`:2603`) — so setting both means only whichever is parsed last wins. + Which one that is is *not* the INI order: `common/preset.cpp` emits `opt.args.back()` while + iterating an unordered map. It happened to resolve to DirectIO, but a container-order change would + silently downgrade to `NONE`, dropping DirectIO *and* mmap for plain buffered reads. `dio` is the + single equivalent of the old pair and also removes two deprecation warnings per launch. Valid + values are `none`, `mmap`, `mlock`, `mmap+mlock`, `dio` (`arg.cpp:2615-2619`) — anything else + throws at startup. + +- **`load-mode = dio` does not enable DirectIO on Windows — it only disables mmap.** The Win32 `llama_file::impl` ctor takes `use_direct_io` as `[[maybe_unused]]` and just calls `ggml_fopen` (`vendor/llama.cpp/src/llama-mmap.cpp:86-95`); `FILE_FLAG_NO_BUFFERING` is never set and `read_alignment()` stays 1 (`:391`), so the loader's async staging buffers are 4 x 1 MiB of pinned host memory instead of the 4 x 64 MiB the aligned path would use (`src/llama-model-loader.cpp:1418`, `:1427`). `has_direct_io()` nevertheless returns a hardcoded `true` on Windows (`:173-175`). Net effect of `dio` on this platform: buffered reads, no mmap, and zero VRAM cost — it is never implicated in a CUDA OOM. Keep the key for the deprecation-warning reason documented above, but do not reason about page-cache behaviour from it. + +## mmproj-offload + +- **`mmproj-offload = true` fails silently at startup on a saturated GPU.** CLIP's warmup + compute buffer OOMs but the server keeps running — only image requests error at generation + time. Set `false` on tiers where LLM + KV already saturate VRAM. + +## Context size and override-kv + +- **`ctx-size` above the GGUF's `context_length` is dead VRAM unless `override-kv` lifts it too.** + `llama-context.cpp:131` never clamps `n_ctx`, so the KV cache really is allocated at the + requested size — but the server then caps every slot at `n_ctx_train` + (`tools/server/server-context.cpp:1201-1203`, applied as `slot.n_ctx = n_ctx_slot` at `:1255`) + and rejects any larger request outright at `:3100` / `:3111`. Both Muse Glimmer GGUFs ship + `muse-glimmer.context_length = 131072`, so the 24 GB entry's former `ctx-size = 262144` + allocated 262144 cells while no request could exceed 131072 — ~884 MiB of unreachable VRAM. + `override-kv = muse-glimmer.context_length=int:262144` raises `n_ctx_train` and is the only + lever; Meta documents 131072 as the default and 262144 as the maximum, so this is the + vendor-sanctioned ceiling, not an extrapolation hack. The startup line + `the slot context (...) exceeds the training context of the model (...) - capping` is expected + here (the pool is 524288 for two 262144 slots); what matters is + `initializing, n_slots = 2, n_ctx_slot = 262144`. If that reads 131072 the override did not + take. It also sets `n_ctx_orig_yarn = 262144` (`src/llama-model.cpp:1180`), inert at + `freq_scale 1`. + +## ngram-mod speculative decoding + +**ngram-mod speculative decoding** (`--spec-type ngram-mod`): model-agnostic, works on any model. +- All models: `spec-ngram-mod-n-match = 24`, `spec-ngram-mod-n-min = 48`, `spec-ngram-mod-n-max = 64` + (matches the struct defaults in `common/common.h:329-337` and what `--spec-default` produces + at `common/arg.cpp:4065-4074`; ggerganov confirmed post-merge in PR #19164 that the min/max + "likely don't need to be changed from the recommended values"; MoEs require long drafts and + dense models tolerate them without noticeable cost). Flags were renamed from + `--draft-min`/`--draft-max`/`--spec-ngram-size-n` in upstream PR #22397; the old names now + error at startup. +- `n_match < 16` logs a "too small — poor quality is possible" warning at + `vendor/llama.cpp/common/speculative.cpp:1031-1034`; parser accepts `1..1024` + (`common/arg.cpp:3606-3615`), so 16 is the lowest non-warning value, not a hard floor. + Min/max parsers accept `0..1024` (`common/arg.cpp:3587-3605`). +- Memory overhead: ~16 MiB **total**, shared across all server slots + (single `common_ngram_mod` instance allocated at `common/speculative.cpp:1026`). +- Pool auto-resets on `begin()` if occupancy > 25 %, and after 3 consecutive rounds with + acceptance < 50 % (`common/speculative.cpp:720-728`, `:790-806`). Smaller `n_match` makes + these resets fire more often and wipes ngrams learned from the current prompt — another + reason to stay at `n_match ≥ 24`. From 10efe9b152f0ba878a1235727f10ef7d8334f877 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sun, 16 Aug 2026 21:15:36 +0200 Subject: [PATCH 09/20] Move build-system rationale from AGENTS.md into docs/build_system.md Condenses Non-obvious behavior to one-line rules, each pointing at the section that now carries its reasoning. Consolidates the three hardcoded vendor paths, previously documented in three separate places, into one post-bump checklist. Notes in the pr-code-review checklist that its restatement of these rules is deliberate, so a review pass stays self-contained. --- .claude/skills/pr-code-review/SKILL.md | 12 ++++-- AGENTS.md | 19 +++++----- CHANGELOG.md | 1 + docs/build_system.md | 51 ++++++++++++++++++++++++++ docs/model_tuning.md | 3 +- 5 files changed, 72 insertions(+), 14 deletions(-) create mode 100644 docs/build_system.md diff --git a/.claude/skills/pr-code-review/SKILL.md b/.claude/skills/pr-code-review/SKILL.md index fa209b3..10d82ad 100644 --- a/.claude/skills/pr-code-review/SKILL.md +++ b/.claude/skills/pr-code-review/SKILL.md @@ -136,7 +136,7 @@ Do NOT flag any of the following: - Pre-existing issues not introduced in this PR's changes. - Code that appears to be a bug but is actually correct. - Pedantic nitpicks that a senior engineer would not flag. -- General code quality concerns unless explicitly required in AGENTS.md. +- General code quality concerns unless explicitly required in AGENTS.md or `docs/`. - Issues explicitly silenced in code (e.g., via a lint ignore comment). - Pure code style or formatting preferences. - Potential issues that depend on specific inputs or runtime state. @@ -148,8 +148,14 @@ Do NOT flag any of the following: These are architectural and safety concerns specific to this PowerShell wrapper around llama.cpp; there is no linter to catch them. Check for these during all passes, in addition to general defect scanning. They are NOT style issues; they -are correctness and safety rules. Each maps to a rule in AGENTS.md -("Non-obvious behavior", "Presets", "Changelog style"). +are correctness and safety rules. + +This checklist deliberately restates rules whose full rationale lives in +`docs/build_system.md`, `docs/presets.md` and `docs/model_tuning.md`, and whose +one-line form lives in AGENTS.md ("Non-obvious behavior", "Traps", "Changelog +style"). The duplication is intentional: a review pass must stay self-contained +and must not depend on on-demand doc reads mid-pass. When a rule changes, all +three locations move together. - **Ephemeral submodule edits:** Changes under `vendor/llama.cpp/` that expect to persist. Each `rebuild_llama.cpp.ps1` resets the submodule to diff --git a/AGENTS.md b/AGENTS.md index c53d6e3..6b3261b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ Canonical agent-instruction file for this repository. Both Claude Code (via the ## What this is -A PowerShell wrapper around upstream [llama.cpp](https://github.com/ggml-org/llama.cpp), pinned as a submodule at `vendor/llama.cpp/`. No original C/C++ lives here — only `.ps1` scripts driving CMake + MSVC + Conda. The session shell is bash on Windows; the project's own scripts must run from `pwsh`/`powershell`. +A PowerShell wrapper around upstream [llama.cpp](https://github.com/ggml-org/llama.cpp), pinned as a submodule at `vendor/llama.cpp/`. No original C/C++ lives here — only `.ps1` scripts driving CMake + MSVC + Conda, plus one benchmark helper in `examples/mtp-bench.py`. The session shell is bash on Windows; the project's own scripts must run from `pwsh`/`powershell`. ## Commands @@ -25,16 +25,16 @@ Binaries land in `./vendor/llama.cpp/build/bin/Release/`. Conda env `llama.cpp` ## Non-obvious behavior -- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt` (idempotent; workaround for `find_package(BLAS)` failing on Windows). `.gitmodules` sets `ignore = dirty` for this reason — don't "clean it up." -- **Each build wipes the submodule** back to `origin/master` then checks out the requested `-version` / PR. Any local edits under `vendor/llama.cpp/` are lost by design. The reset/`--remote` step is **scoped to `vendor/llama.cpp` only** — other submodules (e.g. `vendor/Qwen-Fixed-Chat-Templates`, default branch `main`) stay at the SHA pinned in the superproject and are never advanced by the build script. To bump them, do it manually: `git -C vendor/Qwen-Fixed-Chat-Templates fetch && git -C vendor/Qwen-Fixed-Chat-Templates checkout && git add vendor/Qwen-Fixed-Chat-Templates && git commit`. Once the pin is committed, the next `rebuild_llama.cpp.ps1` mirrors it into the working tree (auto-discovered from `.gitmodules`, `--force`); hand-edits inside the submodule do not survive a rebuild. -- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`.** Upstream `ggml/CMakeLists.txt` sets `cmake_policy(SET CMP0194 NEW)` and declares `project(... ASM)`; on CMake 4.1+ with the VS generator this rejects `cl.exe` as the ASM compiler. The script locates `ml64.exe` via `vswhere.exe`. Don't remove. The `vswhere` call passes `-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64` alongside `-latest` — this is deliberate, not redundant: `-latest` alone returns the newest-installed instance by timestamp, which on a machine with multiple instances may be a Build Tools install lacking the C++ workload, so `-find` returns nothing and the build throws "ml64.exe not found" even though another instance (e.g. Community) has it (#3). `-requires` narrows `-latest` to instances that actually carry the MSVC x64 toolset, matching the pattern upstream uses in `.github/workflows/build-cpu.yml`. Don't drop the `-requires` filter. +- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an idempotent OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt`; `.gitmodules` sets `ignore = dirty` for it. Don't "clean it up." `docs/build_system.md` -> *Submodule lifecycle* +- **Each build wipes `vendor/llama.cpp`** back to `origin/master` then checks out the requested `-version` / PR, so local edits there are lost by design. Other submodules are never advanced by the build script and must be bumped by hand. `docs/build_system.md` -> *Submodule lifecycle* +- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`**, located via `vswhere.exe` with both `-latest` and `-requires ...VC.Tools.x86.x64`. Don't remove either. `docs/build_system.md` -> *Toolchain detection* - **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. -- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and the dispatcher returns `BEST_FATTN_KERNEL_NONE` -> `GGML_ABORT` for any other K type or any mismatched K/V pair (`ggml-cuda/fattn.cu:424-446`). The flag pulls in the full `fattn-vec*.cu` set so asymmetric / q5 / q4_1 caches work (the presets use `q5_0` K + `q4_1` V). Costs extra `nvcc` compile time. -- **Build parallelism is SMT-aware.** `cmake --build --parallel` is fed a count derived from `Win32_Processor`. Upstream's `UseMultiToolTask=true` + `EnforceProcessCountAcrossBuilds=true` (`vendor/llama.cpp/CMakeLists.txt:92-93`) makes this the single cap on concurrent `cl.exe`/`nvcc` — no per-project `/MP` multiplication. On SMT CPUs it uses physical cores (`Sum(NumberOfCores)`): dropping the logical siblings avoids starving the scheduler / ~doubling peak `nvcc` RAM (no throughput gain) and leaves them free so the machine stays usable. On non-SMT CPUs where physical == logical (hybrid Arrow/Lunar Lake; e.g. Core Ultra 9 285HX = 8P+16E, 24 threads) using all cores would peg the box at 100%, so it backs off to 80% of physical (`floor(cores * 0.8)` = 19 on the 285HX) to keep the machine usable during builds. Override with `-parallelJobs N`. -- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`.** It pins `torch` to a CUDA 12.6 wheel, adds `tiktoken` (missing upstream, required for GLM), pins `transformers==5.3.0`, and narrows `numpy` to resolve an `opencv-python-headless` conflict. When bumping any of these, verify both constraints still hold. -- **`server.ps1` reads GGUF metadata** by shelling out to `vendor/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py`. Upstream has moved this path before (CHANGELOG 1.24.0) — if server startup fails with "Failed to extract model details", check the path first. +- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it any asymmetric or non-`f16`/`q4_0`/`q8_0`/`bf16` KV pair aborts at runtime, and the presets use `q5_0` K + `q4_1` V. Costs extra `nvcc` time. `docs/build_system.md` -> *CUDA build flags* +- **Build parallelism is SMT-aware** and is the single cap on concurrent `cl.exe`/`nvcc`: physical cores on SMT CPUs, 80% of them on non-SMT hybrids. Override with `-parallelJobs N`. `docs/build_system.md` -> *Build parallelism* +- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`**, pinning `torch` (cu126), `transformers`, `numpy`, and adding `tiktoken`. When bumping any of them, verify both constraints still hold. `docs/build_system.md` -> *Python requirements layering* +- **Three vendored paths are hardcoded** (`gguf_dump.py`, `speed-bench/`, `models/templates/`). Upstream has moved them before; after a version bump treat a startup failure naming one as a relocation first. `docs/build_system.md` -> *Upstream path dependencies* - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. -- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. If startup fails reading the script after a rebuild, check whether upstream moved `tools/server/bench/speed-bench/` (same failure mode as the `gguf_dump.py` note above). +- **`speed-bench.ps1` drives a router-mode server**, not a single model, and needs the `datasets` package plus network access. It does not work against a plain single-model server. `docs/build_system.md` -> *Upstream path dependencies* - **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. ## Presets @@ -77,5 +77,6 @@ Deep reference documentation lives under `docs/` and is **read on demand**, not | Document | When to read | | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs/build_system.md` | Why the build scripts do what they do: submodule lifecycle, `ml64.exe`/`vswhere` toolchain detection, CUDA flags, SMT-aware parallelism, the Python requirements layering, and the hardcoded upstream paths. **Read before editing `rebuild_llama.cpp.ps1` or any `examples/*.ps1`.** | | `docs/presets.md` | Cross-model INI rules: device pinning and multi-GPU, `load-mode`, `mmproj-offload`, context size and `override-kv`, ngram-mod speculative decoding. **Read before editing any file under `presets/`.** | | `docs/model_tuning.md` | Per-family rationale and measured VRAM/throughput numbers for Qwen 3.6 and 3.8, gemma-4, Bonsai and DSpark, DeepSeek-V4-Flash, and Muse Glimmer. **Read before adding, retuning or removing a model entry.** | diff --git a/CHANGELOG.md b/CHANGELOG.md index f331166..15cfd82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md +- [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md - [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md - [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 diff --git a/docs/build_system.md b/docs/build_system.md new file mode 100644 index 0000000..db6c719 --- /dev/null +++ b/docs/build_system.md @@ -0,0 +1,51 @@ +# Build System Reference + +Why `rebuild_llama.cpp.ps1` and the `examples/*.ps1` scripts do what they do. Not +auto-loaded into the agent context; read it on demand. The one-line rules these +sections back are in AGENTS.md "Non-obvious behavior". + +## Submodule lifecycle + +- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt` (idempotent; workaround for `find_package(BLAS)` failing on Windows). `.gitmodules` sets `ignore = dirty` for this reason — don't "clean it up." + +- **Each build wipes the submodule** back to `origin/master` then checks out the requested `-version` / PR. Any local edits under `vendor/llama.cpp/` are lost by design. The reset/`--remote` step is **scoped to `vendor/llama.cpp` only** — other submodules (e.g. `vendor/Qwen-Fixed-Chat-Templates`, default branch `main`) stay at the SHA pinned in the superproject and are never advanced by the build script. To bump them, do it manually: `git -C vendor/Qwen-Fixed-Chat-Templates fetch && git -C vendor/Qwen-Fixed-Chat-Templates checkout && git add vendor/Qwen-Fixed-Chat-Templates && git commit`. Once the pin is committed, the next `rebuild_llama.cpp.ps1` mirrors it into the working tree (auto-discovered from `.gitmodules`, `--force`); hand-edits inside the submodule do not survive a rebuild. + +## Toolchain detection + +- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`.** Upstream `ggml/CMakeLists.txt` sets `cmake_policy(SET CMP0194 NEW)` and declares `project(... ASM)`; on CMake 4.1+ with the VS generator this rejects `cl.exe` as the ASM compiler. The script locates `ml64.exe` via `vswhere.exe`. Don't remove. The `vswhere` call passes `-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64` alongside `-latest` — this is deliberate, not redundant: `-latest` alone returns the newest-installed instance by timestamp, which on a machine with multiple instances may be a Build Tools install lacking the C++ workload, so `-find` returns nothing and the build throws "ml64.exe not found" even though another instance (e.g. Community) has it (#3). `-requires` narrows `-latest` to instances that actually carry the MSVC x64 toolset, matching the pattern upstream uses in `.github/workflows/build-cpu.yml`. Don't drop the `-requires` filter. + +- **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. + +## CUDA build flags + +- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and the dispatcher returns `BEST_FATTN_KERNEL_NONE` -> `GGML_ABORT` for any other K type or any mismatched K/V pair (`ggml-cuda/fattn.cu:424-446`). The flag pulls in the full `fattn-vec*.cu` set so asymmetric / q5 / q4_1 caches work (the presets use `q5_0` K + `q4_1` V). Costs extra `nvcc` compile time. + +## Build parallelism + +- **Build parallelism is SMT-aware.** `cmake --build --parallel` is fed a count derived from `Win32_Processor`. Upstream's `UseMultiToolTask=true` + `EnforceProcessCountAcrossBuilds=true` (`vendor/llama.cpp/CMakeLists.txt:92-93`) makes this the single cap on concurrent `cl.exe`/`nvcc` — no per-project `/MP` multiplication. On SMT CPUs it uses physical cores (`Sum(NumberOfCores)`): dropping the logical siblings avoids starving the scheduler / ~doubling peak `nvcc` RAM (no throughput gain) and leaves them free so the machine stays usable. On non-SMT CPUs where physical == logical (hybrid Arrow/Lunar Lake; e.g. Core Ultra 9 285HX = 8P+16E, 24 threads) using all cores would peg the box at 100%, so it backs off to 80% of physical (`floor(cores * 0.8)` = 19 on the 285HX) to keep the machine usable during builds. Override with `-parallelJobs N`. + +## Python requirements layering + +- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`.** It pins `torch` to a CUDA 12.6 wheel, adds `tiktoken` (missing upstream, required for GLM), pins `transformers==5.3.0`, and narrows `numpy` to resolve an `opencv-python-headless` conflict. When bumping any of these, verify both constraints still hold. + +## Upstream path dependencies + +This repo hardcodes three paths inside `vendor/llama.cpp/`. Each build resets the submodule to the requested revision, so an upstream move breaks them at runtime with no build-time warning. After any `-version` / `-pullRequest` bump, a startup failure mentioning one of these should be treated as a relocation until proven otherwise: + +| Path | Used by | Failure signature | +| ---- | ------- | ----------------- | +| `gguf-py/gguf/scripts/gguf_dump.py` | `examples/server.ps1` metadata read | `Failed to extract model details` | +| `tools/server/bench/speed-bench/speed_bench.py` | `examples/speed-bench.ps1` | script not found at startup | +| `models/templates/google-gemma-4-31B-it.jinja` | gemma-4 preset entries | template read fails at model load | + +- **`server.ps1` reads GGUF metadata** by shelling out to `vendor/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py`. Upstream has moved this path before (CHANGELOG 1.24.0) — if server startup fails with "Failed to extract model details", check the path first. + +- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. If startup fails reading the script after a rebuild, check whether upstream moved `tools/server/bench/speed-bench/` (same failure mode as the `gguf_dump.py` note above). + +## Script argument parsing + +- **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. + +## Build safety checks + +- **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. diff --git a/docs/model_tuning.md b/docs/model_tuning.md index 5c3564b..1fe6fa8 100644 --- a/docs/model_tuning.md +++ b/docs/model_tuning.md @@ -143,8 +143,7 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. Gemma 4 is trained that way — so cross-turn KV-prefix invalidation is inherent (`ctx-checkpoints` mitigates); do not add a preserve-thinking hack. If startup fails reading the template after a rebuild, check whether upstream moved - `models/templates/` (same failure mode as the `gguf_dump.py` note in AGENTS.md - "Non-obvious behavior"). + `models/templates/` — see `docs/build_system.md` -> *Upstream path dependencies*. ## Bonsai and DSpark From 14bcc935dc614ff3df5d194775d9cce316339979 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sun, 16 Aug 2026 21:18:20 +0200 Subject: [PATCH 10/20] Inject SESSION_ID at session start for both agent clients AGENTS.md has required scratch files under .tmp/sessions// without anything supplying the id, so the existing session directories use three different ad-hoc naming schemes. Claude Code gets a SessionStart hook; OpenCode gets a plugin on experimental.chat.system.transform. The hook uses Windows PowerShell to parse the stdin payload rather than jq, which is not present on this platform. It is invoked through cmd.exe, where it is verified working; the bash on PATH here is WSL and cannot reach either PowerShell binary. The plugin needs no package.json because it imports nothing. Also stops tracking .claude/settings.local.json, which is machine-local. --- .claude/settings.json | 16 ++++++++++++++++ .claude/settings.local.json | 9 --------- .gitignore | 4 ++++ .opencode/plugins/session-id-injector.js | 22 ++++++++++++++++++++++ AGENTS.md | 2 +- CHANGELOG.md | 2 ++ 6 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 .claude/settings.json delete mode 100644 .claude/settings.local.json create mode 100644 .opencode/plugins/session-id-injector.js diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..f4bae0b --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|compact", + "hooks": [ + { + "type": "command", + "command": "powershell -NoProfile -Command \"'SESSION_ID=' + ([Console]::In.ReadToEnd() | ConvertFrom-Json).session_id\"" + } + ] + } + ] + } +} diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index e4dcacb..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(dir /b)", - "Bash(dir)", - "Bash(cat:*)" - ] - } -} diff --git a/.gitignore b/.gitignore index 1be1cc1..eb4610a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ # Local scratch directory for all agent-generated artifacts (screenshots, # diffs, trace outputs, experimental scripts). See AGENTS.md "Scratch Files". .tmp/ + +# Machine-local agent settings. The checked-in counterpart is +# .claude/settings.json. +.claude/settings.local.json diff --git a/.opencode/plugins/session-id-injector.js b/.opencode/plugins/session-id-injector.js new file mode 100644 index 0000000..c3eacb0 --- /dev/null +++ b/.opencode/plugins/session-id-injector.js @@ -0,0 +1,22 @@ +// Injects "SESSION_ID=" into the agent's system prompt every turn via +// OpenCode's experimental.chat.system.transform hook. The agent uses this +// value as in .tmp/sessions// per AGENTS.md +// "Scratch Files". The Claude Code equivalent is the SessionStart hook in +// .claude/settings.json. +// +// No package.json is needed: this file imports nothing, and OpenCode loads +// local plugins directly from .opencode/plugins/. A package.json would only +// be required to pull in external npm packages. +// +// The "experimental." prefix means OpenCode reserves the right to rename or +// remove the hook; sessionID is also optional in its signature, hence the +// guard. If either changes, the soft-fallback in AGENTS.md (mint +// YYYYMMDD-HHMMSS-) keeps things working at the cost of resume +// support. Verified against @opencode-ai/plugin 1.3.17 / opencode-ai 1.18.18. + +export const SessionIdInjector = async () => ({ + 'experimental.chat.system.transform': async (input, output) => { + if (!input.sessionID) return; + output.system.push(`SESSION_ID=${input.sessionID}`); + }, +}); diff --git a/AGENTS.md b/AGENTS.md index 6b3261b..bcaffcb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,7 @@ Prohibitions that cause a silent OOM, silent corruption, or a startup abort. Eac ## Scratch Files -Non-committed agent artifacts (diffs, trace outputs, generated reports, experimental scripts) go under `.tmp/sessions//` at the repo root; `.tmp/` is gitignored. `` is `SESSION_ID` when the platform injects it, otherwise a minted `YYYYMMDD-HHMMSS-`. Never write scratch files to `.claude/`, the repo root, or `vendor/`. +Non-committed agent artifacts (diffs, trace outputs, generated reports, experimental scripts) go under `.tmp/sessions//` at the repo root; `.tmp/` is gitignored. `` is the `SESSION_ID` injected into context at session start — by the `SessionStart` hook in `.claude/settings.json` under Claude Code, by `.opencode/plugins/session-id-injector.js` under OpenCode. If neither fired and no `SESSION_ID` is in context, mint `YYYYMMDD-HHMMSS-` instead. Never write scratch files to `.claude/`, the repo root, or `vendor/`. ## Reference diff --git a/CHANGELOG.md b/CHANGELOG.md index 15cfd82..51df3d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md - [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md - [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md +- [Agents] Inject SESSION_ID at session start for Claude Code and OpenCode +- [Agents] Stop tracking .claude/settings.local.json - [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 From 50d34d644facd3121f4610fbd6c3a9c84f228f7d Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sun, 16 Aug 2026 21:20:19 +0200 Subject: [PATCH 11/20] Add session-retrospective skill Ports the routing-plus-gauntlet retro from the fertilizer_management repo so the new docs/ layout has a maintainer. Routing targets are this repo's files, and the table records that traps live in three places by design so a candidate proposes the edit to all of them at once. The process-worked gauntlet test carries a scope note: with no tests, linter or CI here, the only automated gates are the build and llama.cpp's own startup validation, so the test kills far fewer candidates than it does upstream. --- .claude/skills/session-retrospective/SKILL.md | 241 ++++++++++++++++++ CHANGELOG.md | 7 +- 2 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 .claude/skills/session-retrospective/SKILL.md diff --git a/.claude/skills/session-retrospective/SKILL.md b/.claude/skills/session-retrospective/SKILL.md new file mode 100644 index 0000000..9d1f304 --- /dev/null +++ b/.claude/skills/session-retrospective/SKILL.md @@ -0,0 +1,241 @@ +--- +name: session-retrospective +description: > + End-of-session retrospective. Use when the user asks to review the + current session for improvement opportunities, or types + /session-retrospective. Scans the conversation for friction and discoveries, + classifies each candidate by destination (AGENTS.md, docs/, new + skill, existing skill, drop), runs an adversarial gauntlet that + kills candidates failing already-captured / counterfactual / + generalization / triviality / process-worked tests, then presents + only survivors interactively. Default expected count is zero. + Single-session scope; no cross-session memory. +disable-model-invocation: true +--- + +# Skill: Session Retrospective + +End-of-session retrospective that proposes additions to `AGENTS.md`, +`docs/`, an existing skill, or a new skill, but only when an +adversarial gauntlet has killed every candidate that does not survive +five concrete tests. Default expected output: zero additions. + +## Core principle + +The skill argues against itself before bothering the user. The +default-on bias of an LLM proposing documentation is to over-propose: +"plausible improvements" rather than "improvements that would have +prevented this session's friction". Without an adversarial pass the +user has to do the culling manually with a single dismissive question; +the gauntlet automates that pushback. + +A finding survives only when it passes **all five** gauntlet tests. +The agent must justify each survivor *up* from zero with concrete +evidence, not *down* from a cap. + +The `process-worked` test specifically asks whether an **automated** +process caught the issue, not whether the user did. Treating a human +"wait, why are you doing X?" as `process-worked` collapses the gauntlet +into "the user can override anything, so propose nothing", which is the +failure mode the core principle warns about. The user catching the +issue is evidence the issue is real, not evidence the system already +handled it. + +## When to use + +- The user asks for a retro of the current session. +- The user types `/session-retrospective`. + +Do **not** use this skill: + +- Mid-session ("we just learned X — write it down" is a normal Edit, + not a retro). +- Across multiple sessions. Single-session scope; no friction log, no + recurrence tracking, no cross-session memory. + +## Workflow + +Track the steps with `TodoWrite`. Mark each `in_progress` when +starting and `completed` when done. + +1. **Resolve context.** + - Current branch via `git branch --show-current`. This repo has no + ticket-key convention; the branch name is the report label, with + `/` replaced by `-`. + - Timestamp: `Get-Date -Format "yyyyMMdd-HHmm"`. +2. **Scan session.** Walk the conversation in order. Tag every event + that is one of: + - **Friction:** user corrected output, asked to redo, expressed + frustration, overrode an approach, or 3+ rounds on the same + topic. Distinguish friction from normal iteration ("try a smaller + ctx-size" is collaboration, not friction). + - **Discovery:** a fact about this wrapper, the build, or a model's + behaviour that was learned mid-session and is not yet captured. + - **Procedure:** a multi-step playbook (3+ steps with decision + logic) that was executed and is likely to recur. + - **Skill gap:** a task adjacent to an existing skill where the + skill could have helped but didn't, or didn't exist. +3. **Classify by destination** using the routing table below. +4. **Self-challenge gauntlet.** Run every candidate through all five + tests below. A candidate survives only when it passes all five. + Maintain a kill log throughout: every killed candidate with the + test that killed it and, where relevant, the existing line that + already covers the fact. +5. **Prioritize survivors.** Default expected count is **zero**. Hard + cap **3**. Each survivor must come with a one-sentence statement + of the future-session mistake it prevents. If that sentence cannot + be written cleanly, the candidate goes back through the gauntlet + and almost always dies. +6. **Read existing destinations** for each survivor to detect last-mile + duplicates, contradictions, and the right insertion point. +7. **Present interactively, one at a time.** For each survivor: + - Quote the session evidence. + - Show the proposed concrete diff (exact text, exact location). + - State the future-session mistake it prevents. + - Ask confirm / reject / refine. +8. **Apply confirmed edits** via `Edit` / `Write`. Skip rejected ones. +9. **Write the report** to + `.tmp/sessions//retro-.md`. Report shape is in + the *Report* section below. +10. **Print the terminal output** of the turn. Exactly one of: + - `No additions needed.` + the kill log + the report file link. + - `N proposed addition(s).` (1–3) + the interactive presentation + + the kill log of dropped candidates + the report file link. + + The report file link is the last thing printed. Nothing follows + it. Any follow-up question opens a new turn. + +## Routing table + +| Candidate signal | Destination | +| --------------------------------------------------------------------------- | ------------------------------------------------------ | +| Behavioural rule about the build, the scripts, or the repo layout | `AGENTS.md` -> *Non-obvious behavior* | +| Prohibition whose violation is a silent OOM, silent corruption, or an abort | `AGENTS.md` -> *Traps*, **and** the backing `docs/` section | +| Build-script, toolchain, submodule, or vendored-path reasoning | `docs/build_system.md` | +| Cross-model INI rule (device pinning, `load-mode`, context sizing, spec) | `docs/presets.md` | +| Per-model rationale or a measured VRAM / throughput number | `docs/model_tuning.md` | +| User-facing launch or INI-syntax instruction | `presets/README.md` | +| 3+ step playbook with decision logic, fits an existing skill's scope | Extend `.claude/skills//SKILL.md` | +| 3+ step playbook that's its own concern | New `.claude/skills//SKILL.md` | +| Tool/MCP gap | Note in report only; no auto-edit | +| Session-specific minutiae unlikely to generalize | **Drop**; note in kill log | + +Some rules deliberately live in more than one place. A trap is stated +in `AGENTS.md` *Traps* without rationale, in its `docs/` section with +rationale, and — where a reviewer needs it without an on-demand read — +in the `pr-code-review` *Project-Specific Review Checklist*. When a +candidate targets one of those, propose the edit to **all** copies in a +single survivor, never to one in isolation. + +## Self-challenge gauntlet + +A candidate survives only when it passes **all five** tests. + +| Test | Operationalized as | Kill if | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| Already-captured | Run `rg -i "" AGENTS.md docs/ presets/README.md .claude/skills/` and `git log -p -n 20 -- AGENTS.md docs/ .claude/skills/` for in-session commits. | Any hit covers the same fact. Quote the existing line in the kill log. | +| Counterfactual | Locate the actual mistake event. Did the user correct a fact the agent **didn't know**, or one it **knew but lapsed on**? | If lapse, kill: more docs don't fix lapses. | +| Generalization | Would this be equally true of any llama.cpp build, or does it depend on this Windows / MSVC / CUDA / Conda wrapper and its pinned presets? | If it holds anywhere, kill: generic llama.cpp or PowerShell knowledge. | +| Triviality | Can any agent derive this in <60 seconds from `--help`, the INI files, or the build script? | If yes, kill: permanent context noise must clear a higher bar. | +| Process-worked | Did an **automated** process catch the issue *in this session*? See the scope note below — it is narrow in this repo. The user reading agent output and pushing back does NOT count. | If yes, kill: the existing automated process is the doc. | + +**Scope note on `process-worked`.** This repo has no tests, no linter, +no type-checker and no CI (`AGENTS.md` -> *Commands*). The only automated +gates are the build itself (`rebuild_llama.cpp.ps1`), the running +build-tree-process abort, and llama.cpp's own startup validation — a +refused `cache-type` pair, an unknown `load-mode` value, a missing +template file. This test therefore kills far fewer candidates here than +in a repo with a test suite; do not stretch it to cover a human review. +Conversely, a failure mode that llama.cpp *silently* accepts is by +definition not caught by any process, and is exactly what *Traps* exists +for. + +If the agent cannot identify which test a candidate would *fail*, the +candidate is not yet justified; kill it. + +## Friction signals + +| Signal | Example | +| ------------------------ | ---------------------------------------------------------------------- | +| User asks to redo | "No, redo this", "Try again", "That's not what I meant" | +| User corrects output | "Actually it should be X", "Change this to Y" | +| User pushback | "Is this really adding value?", "Keep it simple", "You keep doing X" | +| Excessive back-and-forth | 3+ rounds on the same topic without resolution | +| User overrides approach | "Don't do it that way", "Skip that step", "Just do X" | + +Do **not** treat as friction: + +- Normal iterative refinement. +- User exploring options. +- User changing direction (new input, not a mistake). +- A tuning loop that converges. Sweeping `ctx-size` or `fit-target` + across several launches is the documented way this repo finds a + number, not evidence of a problem. + +## Report + +Path: `.tmp/sessions//retro-.md`. + +Create `.tmp/sessions//` if it does not exist. Re-running +the retro for the same branch in the same session overwrites the prior +report; that is intentional. Branch names lowercase with `/` replaced +by `-`. + +Body: + +```markdown +# Session Retro: () + +## Survivors + + + +### : + +- **Evidence:** +- **Prevents:** +- **Diff:** +- **Decision:** confirmed | rejected | refined + +## Kill log + +| Candidate | Destination | Killed by | Note | +| -------------------- | ----------- | ----------------- | ----------------------------------------------------- | +| | | already-captured | Quote the existing line, or cite the file:line. | +| | | counterfactual | Agent knew the rule; lapse, not knowledge gap. | +| | | generalization | Generic knowledge. | +| | | triviality | Derivable in <60 s from . | +| | | process-worked | Caught in-session by . | + +## Summary + + +``` + +## Hard guarantees + +- Never writes to `AGENTS.md` / `docs/` / `presets/` / any skill + without explicit per-finding user confirmation. +- Never invents friction; uses only observable session evidence. +- Default expected survivor count is **zero**. Hard cap **3**. +- Kill log is mandatory output for every retro, even when zero + candidates survive. The kill log is part of the chat output, not + only the report file. +- Never modifies build scripts or preset INI files; docs and skills + only. A preset value that should change is a finding to report, not + an edit to make. +- Never commits or pushes. +- Single-session scope. No friction log, no recurrence tracking, no + cross-session memory. +- Report files only under `.tmp/sessions//`. Never + `.claude/`, the repo root, or `vendor/`. + +## Constraints + +- Do NOT modify source code (docs and skills only). +- Do NOT post comments, reviews, or any data to GitHub. +- Do NOT commit or push. +- Do NOT silently surface candidates that failed the gauntlet, even + if they look interesting. The user can override a kill by name + after seeing the kill log; the skill does not relitigate on its + own. diff --git a/CHANGELOG.md b/CHANGELOG.md index 51df3d8..10a7934 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,13 +19,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document Muse-Glimmer-30B RoPE, swa-full, and context-shift behavior - [Documentation] Document the Muse-Glimmer-30B DFlash drafter, chat template, and reasoning strength defaults - [Documentation] Document the measured Muse-Glimmer-30B 24 GB compute buffer and VRAM breakdown +- [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md +- [Agents] Add session-retrospective skill for routing session learnings into AGENTS.md and docs/ +- [Agents] Inject SESSION_ID at session start for Claude Code and OpenCode ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md - [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md -- [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md -- [Agents] Inject SESSION_ID at session start for Claude Code and OpenCode -- [Agents] Stop tracking .claude/settings.local.json - [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 @@ -33,6 +33,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier +- [Agents] Stop tracking .claude/settings.local.json ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature From 3cf64b19db5e3b5d19f8f1df698ba0ab9d40437d Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Sun, 16 Aug 2026 21:33:41 +0200 Subject: [PATCH 12/20] Trim AGENTS.md to the rules that fire regardless of task Measured against the last 40 commits: presets/ is touched by 24, the llama.cpp submodule pointer moves in 6, rebuild_llama.cpp.ps1 is edited in 2, examples/ in none. Non-obvious behavior was spending always-on context on build-configuration detail at that bottom rate, so the five summaries whose full text already sits in docs/build_system.md are dropped; the Reference trigger already names every one. Adds the three examples scripts that were never named anywhere, points at README.md Usage as the end-user inventory instead of restating flags, and records which scripts actually answer Get-Help. Also removes the gguf_dump.py bullet that restated its own table row. --- AGENTS.md | 9 +++------ CHANGELOG.md | 2 ++ docs/build_system.md | 6 ++++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bcaffcb..da3deb5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,20 +21,17 @@ Get-Help -Detailed ./examples/server.ps1 # full option list Binaries land in `./vendor/llama.cpp/build/bin/Release/`. Conda env `llama.cpp` (Python 3.12) must already exist — the scripts call `conda activate llama.cpp` themselves. +Other helpers in `examples/`: `count_tokens.ps1`, `benchmark.ps1` (perplexity), `speculative_decoding.ps1`, `speed-bench.ps1` (router-mode throughput sweep), `mtp-bench.py`. `README.md` -> *Usage* is the end-user inventory of what they do; don't restate their flags here. `Get-Help` works on `server.ps1`, `speed-bench.ps1` and `count_tokens.ps1` only — the other two carry no comment-based help. + **No tests, no linter.** Verify changes by running an example script against a real GGUF model. ## Non-obvious behavior - **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an idempotent OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt`; `.gitmodules` sets `ignore = dirty` for it. Don't "clean it up." `docs/build_system.md` -> *Submodule lifecycle* - **Each build wipes `vendor/llama.cpp`** back to `origin/master` then checks out the requested `-version` / PR, so local edits there are lost by design. Other submodules are never advanced by the build script and must be bumped by hand. `docs/build_system.md` -> *Submodule lifecycle* -- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`**, located via `vswhere.exe` with both `-latest` and `-requires ...VC.Tools.x86.x64`. Don't remove either. `docs/build_system.md` -> *Toolchain detection* - **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. -- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it any asymmetric or non-`f16`/`q4_0`/`q8_0`/`bf16` KV pair aborts at runtime, and the presets use `q5_0` K + `q4_1` V. Costs extra `nvcc` time. `docs/build_system.md` -> *CUDA build flags* -- **Build parallelism is SMT-aware** and is the single cap on concurrent `cl.exe`/`nvcc`: physical cores on SMT CPUs, 80% of them on non-SMT hybrids. Override with `-parallelJobs N`. `docs/build_system.md` -> *Build parallelism* -- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`**, pinning `torch` (cu126), `transformers`, `numpy`, and adding `tiktoken`. When bumping any of them, verify both constraints still hold. `docs/build_system.md` -> *Python requirements layering* - **Three vendored paths are hardcoded** (`gguf_dump.py`, `speed-bench/`, `models/templates/`). Upstream has moved them before; after a version bump treat a startup failure naming one as a relocation first. `docs/build_system.md` -> *Upstream path dependencies* - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. -- **`speed-bench.ps1` drives a router-mode server**, not a single model, and needs the `datasets` package plus network access. It does not work against a plain single-model server. `docs/build_system.md` -> *Upstream path dependencies* - **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. ## Presets @@ -77,6 +74,6 @@ Deep reference documentation lives under `docs/` and is **read on demand**, not | Document | When to read | | ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docs/build_system.md` | Why the build scripts do what they do: submodule lifecycle, `ml64.exe`/`vswhere` toolchain detection, CUDA flags, SMT-aware parallelism, the Python requirements layering, and the hardcoded upstream paths. **Read before editing `rebuild_llama.cpp.ps1` or any `examples/*.ps1`.** | +| `docs/build_system.md` | Why the build scripts do what they do: submodule lifecycle, `ml64.exe`/`vswhere` toolchain detection, CUDA flags, SMT-aware parallelism, the Python requirements layering, the hardcoded upstream paths, and how `speed-bench.ps1` drives a router-mode server. **Read before editing `rebuild_llama.cpp.ps1` or any `examples/*.ps1`, and before running `speed-bench.ps1`.** | | `docs/presets.md` | Cross-model INI rules: device pinning and multi-GPU, `load-mode`, `mmproj-offload`, context size and `override-kv`, ngram-mod speculative decoding. **Read before editing any file under `presets/`.** | | `docs/model_tuning.md` | Per-family rationale and measured VRAM/throughput numbers for Qwen 3.6 and 3.8, gemma-4, Bonsai and DSpark, DeepSeek-V4-Flash, and Muse Glimmer. **Read before adding, retuning or removing a model entry.** | diff --git a/CHANGELOG.md b/CHANGELOG.md index 10a7934..69fec37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md - [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md +- [Documentation] Drop the build-configuration summaries from AGENTS.md that docs/build_system.md already carries +- [Documentation] List the remaining examples scripts and their help coverage in AGENTS.md - [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 diff --git a/docs/build_system.md b/docs/build_system.md index db6c719..60d69fb 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -38,9 +38,11 @@ This repo hardcodes three paths inside `vendor/llama.cpp/`. Each build resets th | `tools/server/bench/speed-bench/speed_bench.py` | `examples/speed-bench.ps1` | script not found at startup | | `models/templates/google-gemma-4-31B-it.jinja` | gemma-4 preset entries | template read fails at model load | -- **`server.ps1` reads GGUF metadata** by shelling out to `vendor/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py`. Upstream has moved this path before (CHANGELOG 1.24.0) — if server startup fails with "Failed to extract model details", check the path first. +This is not hypothetical: upstream already relocated `gguf_dump.py` once, which is what CHANGELOG 1.24.0 records. -- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. If startup fails reading the script after a rebuild, check whether upstream moved `tools/server/bench/speed-bench/` (same failure mode as the `gguf_dump.py` note above). +## speed-bench + +- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. ## Script argument parsing From 33454547932ba6c80e833183fb91be995dffd0d8 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Thu, 27 Aug 2026 18:23:29 +0200 Subject: [PATCH 13/20] Add rebuild patch step and fix the gguf-py row-group conversion slowdown gguf-py's _apply_over_grouped_rows collected every 16-row group in a Python list and merged them with one np.concatenate(..., out=out). That keeps all groups alive at once and costs an extra full copy of the result, and it degrades with tensor size instead of staying linear. Converting Qwen3.8-Flash-Next, the ffn_down_exps tensors (512 x 2560 x 640 f32) ran at 13.2 MB/s while the gate/up_exps layout (512 x 640 x 2560) managed 348.8 MB/s for the same byte count, the only difference being 4x fewer groups. Writing each group straight into out, and sizing groups by bytes so one group stays in L2 at any row length, gives a uniform ~500 MB/s and takes the per-tensor pipeline from 126.7 s to 4.7 s. What is left is the torch.stack upcast, which this does not touch. A second hunk drops a np.uint64 widening in BF16.quantize_blocks, worth a further 21%. The nan fixup directly above it caps n, so the add cannot overflow uint32. Both hunks are byte-for-byte identical to upstream. Verified against the C implementation through gguf-py/tests/test_quants.py (25 types, 60 exact matches, output unchanged apart from a shifted warning line number), across all 24 registered quant types in both quantize and dequantize directions, and for the uint32 kernel exhaustively over all 2^32 float32 bit patterns. The fix ships as ./patches/0001-gguf-py-write-row-groups-directly.patch because each build resets the submodule to origin/master. The new step applies patches after the -version / PR checkout and throws on failure, so an upstream move cannot silently leave an unpatched, slow tree. git apply --3way implies --index, so the step unstages afterwards to keep the submodule showing the same plain worktree modification as the OpenBLAS shim. --- AGENTS.md | 1 + CHANGELOG.md | 6 +++ docs/build_system.md | 8 ++++ ...01-gguf-py-write-row-groups-directly.patch | 37 +++++++++++++++++++ rebuild_llama.cpp.ps1 | 32 ++++++++++++++++ 5 files changed, 84 insertions(+) create mode 100644 patches/0001-gguf-py-write-row-groups-directly.patch diff --git a/AGENTS.md b/AGENTS.md index da3deb5..fb329c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,6 +29,7 @@ Other helpers in `examples/`: `count_tokens.ps1`, `benchmark.ps1` (perplexity), - **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an idempotent OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt`; `.gitmodules` sets `ignore = dirty` for it. Don't "clean it up." `docs/build_system.md` -> *Submodule lifecycle* - **Each build wipes `vendor/llama.cpp`** back to `origin/master` then checks out the requested `-version` / PR, so local edits there are lost by design. Other submodules are never advanced by the build script and must be bumped by hand. `docs/build_system.md` -> *Submodule lifecycle* +- **`./patches/*.patch` is the only supported way to carry a local change into the submodule.** It is re-applied after every checkout and the build aborts if a patch no longer applies. Currently one patch, which keeps GGUF conversion of large short-row tensors from running 26x too slow. `docs/build_system.md` -> *Local patches* - **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. - **Three vendored paths are hardcoded** (`gguf_dump.py`, `speed-bench/`, `models/templates/`). Upstream has moved them before; after a version bump treat a startup failure naming one as a relocation first. `docs/build_system.md` -> *Upstream path dependencies* - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. diff --git a/CHANGELOG.md b/CHANGELOG.md index 69fec37..02dfff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md - [Agents] Add session-retrospective skill for routing session learnings into AGENTS.md and docs/ - [Agents] Inject SESSION_ID at session start for Claude Code and OpenCode +- [Build] Apply patches from ./patches to the llama.cpp submodule after checkout +- [Build] Abort the build when a local patch no longer applies +- [Documentation] Document the local patch mechanism in the build system reference +- [Documentation] Document the gguf-py row-group fix and how it was verified ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md @@ -40,6 +44,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature - [Documentation] Correct the claim that reasoning_effort has no CLI flag +- [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow +- [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer ## [1.40.0] - 2026-08-05 diff --git a/docs/build_system.md b/docs/build_system.md index 60d69fb..b63db81 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -10,6 +10,14 @@ sections back are in AGENTS.md "Non-obvious behavior". - **Each build wipes the submodule** back to `origin/master` then checks out the requested `-version` / PR. Any local edits under `vendor/llama.cpp/` are lost by design. The reset/`--remote` step is **scoped to `vendor/llama.cpp` only** — other submodules (e.g. `vendor/Qwen-Fixed-Chat-Templates`, default branch `main`) stay at the SHA pinned in the superproject and are never advanced by the build script. To bump them, do it manually: `git -C vendor/Qwen-Fixed-Chat-Templates fetch && git -C vendor/Qwen-Fixed-Chat-Templates checkout && git add vendor/Qwen-Fixed-Chat-Templates && git commit`. Once the pin is committed, the next `rebuild_llama.cpp.ps1` mirrors it into the working tree (auto-discovered from `.gitmodules`, `--force`); hand-edits inside the submodule do not survive a rebuild. +## Local patches + +- **`./patches/*.patch` are re-applied to `vendor/llama.cpp` on every build.** The step runs after the `-version` / PR checkout (`rebuild_llama.cpp.ps1:254-284`), because the checkout would otherwise discard them. It needs no idempotence guard like the OpenBLAS shim — the `reset --hard` earlier in the script guarantees a clean tree. Applied with `git apply --3way`, so a patch still lands when upstream moves the lines *around* a hunk; if it fails, upstream moved the patched code itself and the script `throw`s rather than silently building an unpatched tree. `--3way` implies `--index`, which would leave the files staged and make `git checkout -- ` restore the *patched* copy instead of the upstream one — the trailing `git reset --quiet` unstages them so they show up as plain worktree modifications, same as the shim. + +- **`0001-gguf-py-write-row-groups-directly.patch` fixes a 26x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) collected every 16-row group in a Python list and merged them with a single `np.concatenate(..., out=out)`. That keeps all groups alive at once and costs an extra full copy of the result, and it degrades sharply with tensor size rather than linearly. Measured on Qwen3.8-Flash-Next `blk.N.ffn_down_exps.weight` (512 x 2560 x 640 f32, 1.68 GB out), BF16 conversion ran at **13.2 MB/s** — against 348.8 MB/s for the identical byte count in the `ffn_gate/up_exps` layout (512 x 640 x 2560), which has 4x fewer groups. Writing each group straight into `out`, and sizing groups by bytes (160 KiB, so one group stays in L2 at any row length), gives a uniform ~500 MB/s: 497.9 / 505.9 / 490.8 MB/s on the down, gate-up and `per_layer_token_embd` shapes, i.e. 37.6x / 1.5x / 2.2x. Per expert tensor the whole stack-plus-quantize pipeline goes 126.7 s -> 4.7 s (26.7x); the residual 1.4 s is the `torch.stack` upcast, which this patch does not touch. The second hunk drops a `np.uint64` widening in `BF16.quantize_blocks` (worth a further ~21%); the NaN fixup directly above it caps `n`, so the add cannot overflow uint32. + +- **The patch is byte-for-byte equivalent to upstream, and that was verified three ways.** Against the C implementation with `python gguf-py/tests/test_quants.py --libggml build/bin/Release/ggml-base.dll` (25 types, 60 exact matches, identical output before and after apart from a shifted warning line number); across all 24 registered quant types in both quantize and dequantize directions over six shapes chosen to exercise the remainder path; and for the `uint32` kernel exhaustively over all 2^32 float32 bit patterns (0 mismatches, NaN-fixup path hit 16.7M times). Note `test_quants.py` needs `ggml-base.dll`, not `ggml.dll` — only the former exports `ggml_quantize_chunk` — and `ctypes` needs both `%CUDA_PATH%\bin` and `%CUDA_PATH%\bin\x64` added via `os.add_dll_directory` before the load will resolve. + ## Toolchain detection - **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`.** Upstream `ggml/CMakeLists.txt` sets `cmake_policy(SET CMP0194 NEW)` and declares `project(... ASM)`; on CMake 4.1+ with the VS generator this rejects `cl.exe` as the ASM compiler. The script locates `ml64.exe` via `vswhere.exe`. Don't remove. The `vswhere` call passes `-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64` alongside `-latest` — this is deliberate, not redundant: `-latest` alone returns the newest-installed instance by timestamp, which on a machine with multiple instances may be a Build Tools install lacking the C++ workload, so `-find` returns nothing and the build throws "ml64.exe not found" even though another instance (e.g. Community) has it (#3). `-requires` narrows `-latest` to instances that actually carry the MSVC x64 toolset, matching the pattern upstream uses in `.github/workflows/build-cpu.yml`. Don't drop the `-requires` filter. diff --git a/patches/0001-gguf-py-write-row-groups-directly.patch b/patches/0001-gguf-py-write-row-groups-directly.patch new file mode 100644 index 0000000..6172470 --- /dev/null +++ b/patches/0001-gguf-py-write-row-groups-directly.patch @@ -0,0 +1,37 @@ +diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py +index 80966b6ef..f4495760e 100644 +--- a/gguf-py/gguf/quants.py ++++ b/gguf-py/gguf/quants.py +@@ -33,9 +33,17 @@ def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.n + for dim in oshape: + osize *= dim + out = np.empty(shape=osize, dtype=otype) +- # compute over groups of 16 rows (arbitrary, but seems good for performance) +- n_groups = (rows.shape[0] // 16) or 1 +- np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out) ++ # group by bytes, not by row count, so one group stays in L2 for any row length ++ row_bytes = rows.shape[1] * arr.dtype.itemsize ++ n_per_group = max(1, (160 * 1024) // row_bytes) if row_bytes else 1 ++ # write each group straight into out. collecting the groups for a final ++ # np.concatenate keeps them all alive, which is very slow for big tensors ++ pos = 0 ++ for start in range(0, rows.shape[0], n_per_group): ++ group = func(rows[start:start + n_per_group]).ravel() ++ out[pos:pos + group.size] = group ++ pos += group.size ++ assert pos == osize + return out.reshape(oshape) + + +@@ -209,8 +217,9 @@ class BF16(__Quant, qtype=GGMLQuantizationType.BF16): + n = blocks.view(np.uint32) + # force nan to quiet + n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n) +- # round to nearest even +- n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16 ++ # round to nearest even. the nan fixup above caps n, so this cannot ++ # overflow and does not need to widen the whole array to uint64 ++ n = (n + np.uint32(0x7fff) + ((n >> np.uint32(16)) & np.uint32(1))) >> np.uint32(16) + return n.astype(np.uint16).view(np.uint8) + + @classmethod diff --git a/rebuild_llama.cpp.ps1 b/rebuild_llama.cpp.ps1 index 944a384..2133bae 100644 --- a/rebuild_llama.cpp.ps1 +++ b/rebuild_llama.cpp.ps1 @@ -251,6 +251,38 @@ if (!$pullRequest) { git -C ./vendor/llama.cpp reset --hard PR } +# Local fixes to the vendored tree, re-applied on every build because the +# submodule was reset to origin/master above. Must run after the checkout, +# otherwise the checkout discards them again. +$patches = @(Get-ChildItem -Path "./patches/*.patch" -ErrorAction "SilentlyContinue" | Sort-Object -Property "Name") + +if ($patches.Count -eq 0) { + Write-Host "[Patches] No patches found under ./patches." -ForegroundColor "DarkYellow" +} + +foreach ($patch in $patches) { + + Write-Host "[Patches] Applying $($patch.Name)..." -ForegroundColor "Yellow" + + # --3way still applies when upstream moved the surrounding lines. A failure + # here means upstream moved the patched code itself, so fail loudly instead + # of silently building an unpatched tree. + git -C ./vendor/llama.cpp apply --3way --whitespace=nowarn $patch.FullName + + if ($LASTEXITCODE -ne 0) { + throw "Failed to apply ./patches/$($patch.Name).`nUpstream likely moved the code it targets. Review and regenerate the patch." + } +} + +if ($patches.Count -gt 0) { + + # `git apply --3way` implies `--index` and would leave the patched files + # staged, which makes `git checkout -- ` restore the patched copy + # instead of the upstream one. Unstage so the submodule shows the same + # plain worktree modification as the OpenBLAS shim below. + git -C ./vendor/llama.cpp reset --quiet +} + $lines = @( "# This is a workaround for a CMake bug on Windows to build llama.cpp" "# with OpenBLAS. The find_package(BLAS) call fails to find OpenBLAS," From b3e47a35bdade03e749896995f5efed0989ebcbd Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Thu, 27 Aug 2026 21:19:48 +0200 Subject: [PATCH 14/20] Correct the stated cause of the gguf-py row-group slowdown The patch bullet claimed the old code cost an extra full copy of the result. It does not: np.concatenate(..., out=out) and the slice assignment that replaced it each copy every element exactly once, and both allocate one array per group. What the patch removes is peak intermediate memory, which equalled the whole output, plus the cache miss on re-reading groups that had gone cold. Restated accordingly, and the degradation is now tied to the group count rather than to tensor size, which is what the gate-up versus down comparison actually isolates. The 160 KiB group budget is computed from the input row bytes, so the dequantize direction produces a group several times larger than that. Noted, because the old wording promised L2 residency in both directions. The uint32 overflow argument now carries its bounds instead of asserting the conclusion: the NaN fixup leaves n at most 0xFFFF0000 with a zero low half, everything else is capped by -inf at 0xFF800000, so n + 0x7fff + 1 stays at or below 0xFFFF8000. The C reference gets the same guarantee from an early return that the vectorised form cannot use. A new bullet records that this is not the per_layer_token_embd OOM fix. That table reaches _apply_over_grouped_rows one shard at a time through LazyChunkedTensor, so the function never sees more than about 1.6 GB of it, and peak RSS there was solved by 53c0f624a and a510c82e1. a510c82e1 also landed after the 2.2x figure was measured, so that number needs re-measuring before it is quoted. --- CHANGELOG.md | 2 ++ docs/build_system.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02dfff8..911b85c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Build] Abort the build when a local patch no longer applies - [Documentation] Document the local patch mechanism in the build system reference - [Documentation] Document the gguf-py row-group fix and how it was verified +- [Documentation] Document that the gguf-py row-group fix is a throughput fix and not the PLE table OOM fix ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md @@ -44,6 +45,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature - [Documentation] Correct the claim that reasoning_effort has no CLI flag +- [Documentation] Correct the stated cause of the gguf-py row-group conversion slowdown - [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow - [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer diff --git a/docs/build_system.md b/docs/build_system.md index b63db81..711ee5b 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -14,7 +14,9 @@ sections back are in AGENTS.md "Non-obvious behavior". - **`./patches/*.patch` are re-applied to `vendor/llama.cpp` on every build.** The step runs after the `-version` / PR checkout (`rebuild_llama.cpp.ps1:254-284`), because the checkout would otherwise discard them. It needs no idempotence guard like the OpenBLAS shim — the `reset --hard` earlier in the script guarantees a clean tree. Applied with `git apply --3way`, so a patch still lands when upstream moves the lines *around* a hunk; if it fails, upstream moved the patched code itself and the script `throw`s rather than silently building an unpatched tree. `--3way` implies `--index`, which would leave the files staged and make `git checkout -- ` restore the *patched* copy instead of the upstream one — the trailing `git reset --quiet` unstages them so they show up as plain worktree modifications, same as the shim. -- **`0001-gguf-py-write-row-groups-directly.patch` fixes a 26x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) collected every 16-row group in a Python list and merged them with a single `np.concatenate(..., out=out)`. That keeps all groups alive at once and costs an extra full copy of the result, and it degrades sharply with tensor size rather than linearly. Measured on Qwen3.8-Flash-Next `blk.N.ffn_down_exps.weight` (512 x 2560 x 640 f32, 1.68 GB out), BF16 conversion ran at **13.2 MB/s** — against 348.8 MB/s for the identical byte count in the `ffn_gate/up_exps` layout (512 x 640 x 2560), which has 4x fewer groups. Writing each group straight into `out`, and sizing groups by bytes (160 KiB, so one group stays in L2 at any row length), gives a uniform ~500 MB/s: 497.9 / 505.9 / 490.8 MB/s on the down, gate-up and `per_layer_token_embd` shapes, i.e. 37.6x / 1.5x / 2.2x. Per expert tensor the whole stack-plus-quantize pipeline goes 126.7 s -> 4.7 s (26.7x); the residual 1.4 s is the `torch.stack` upcast, which this patch does not touch. The second hunk drops a `np.uint64` widening in `BF16.quantize_blocks` (worth a further ~21%); the NaN fixup directly above it caps `n`, so the add cannot overflow uint32. +- **`0001-gguf-py-write-row-groups-directly.patch` fixes a 26x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) collected every 16-row group in a Python list and merged them with a single `np.concatenate(..., out=out)`. That holds every group alive until the merge, so peak intermediate memory equals the whole output, and because the group is fixed at 16 *rows* regardless of row length, cost per byte tracks the group count rather than the byte count. The copy count is **not** what changes: both forms allocate one array per group and copy it into `out` once, so the win is peak memory and cache locality, not a saved copy. Measured on Qwen3.8-Flash-Next `blk.N.ffn_down_exps.weight` (512 x 2560 x 640 f32, 1.68 GB out), BF16 conversion ran at **13.2 MB/s** — against 348.8 MB/s for the identical byte count in the `ffn_gate/up_exps` layout (512 x 640 x 2560), which has 4x fewer groups. Writing each group straight into `out`, and sizing groups by bytes (160 KiB of *input*, so one group stays in L2 at any row length; the dequantize direction produces a proportionally larger group), gives a uniform ~500 MB/s: 497.9 / 505.9 / 490.8 MB/s on the down, gate-up and `per_layer_token_embd` shapes, i.e. 37.6x / 1.5x / 2.2x. Per expert tensor the whole stack-plus-quantize pipeline goes 126.7 s -> 4.7 s (26.7x); the residual 1.4 s is the `torch.stack` upcast, which this patch does not touch. The second hunk drops a `np.uint64` widening in `BF16.quantize_blocks` (worth a further ~21%): the NaN fixup directly above it rewrites NaN lanes to `(u.i & 0xffff0000) | 0x00400000`, bounding `n` at `0xFFFF0000` there and at `0xFF800000` (-inf) everywhere else, so `n + 0x7fff + 1 <= 0xFFFF8000` and the add cannot overflow uint32. The C reference (`ggml/src/ggml-impl.h:611-623`) gets the same guarantee from an early `return` that the vectorised form cannot use. + +- **It is a throughput fix, not an OOM fix, and the `per_layer_token_embd` figure is per shard.** `conversion/qwen4exp.py:145-172` wraps the 128 PLE shards in `gguf.LazyChunkedTensor`, whose `tofile` quantizes one shard at a time (`gguf-py/gguf/lazy.py:275-289`), so `_apply_over_grouped_rows` only ever sees ~1.6 GB of that table, never all 95 GiB. Peak RSS for it was solved separately by `53c0f624a` (stream the shards, ~300 GB -> one shard) and `a510c82e1` (`LazyChunkedTensor`) — don't re-credit that to this patch. `a510c82e1` also landed *after* the 2.2x measurement above, so re-measure that shape on the current submodule HEAD before quoting it anywhere, and don't pitch the patch upstream against #15623 / #15648. - **The patch is byte-for-byte equivalent to upstream, and that was verified three ways.** Against the C implementation with `python gguf-py/tests/test_quants.py --libggml build/bin/Release/ggml-base.dll` (25 types, 60 exact matches, identical output before and after apart from a shifted warning line number); across all 24 registered quant types in both quantize and dequantize directions over six shapes chosen to exercise the remainder path; and for the `uint32` kernel exhaustively over all 2^32 float32 bit patterns (0 mismatches, NaN-fixup path hit 16.7M times). Note `test_quants.py` needs `ggml-base.dll`, not `ggml.dll` — only the former exports `ggml_quantize_chunk` — and `ctypes` needs both `%CUDA_PATH%\bin` and `%CUDA_PATH%\bin\x64` added via `os.add_dll_directory` before the load will resolve. From 335892c3c34c5c0875d0ef908db646ddcb3ea8e4 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Thu, 27 Aug 2026 21:21:23 +0200 Subject: [PATCH 15/20] Pin xhigh on Qwen3.8-27B for chat template v22.1 and document the CUDA build flags v22.1 moved the reasoning-effort default from xhigh to medium, so the Qwen3.8-27B entry, which relied on the template default, was silently downgraded at the bump. It now pins xhigh explicitly. model_tuning.md records the new level semantics: medium is the level that injects no instruction text, unknown levels now map down to medium rather than up to xhigh, and a think-tag typed inside a message can override the kwarg. Adds why xhigh is kept rather than lowered, and why --reasoning-budget is the guard rail for it. Qwen publishes no per-level benchmarks and the whole mechanism is one injected sentence, 123 rendered chars against 332. The failure that moved the upstream default to medium was a truncation artifact: with no budget, xhigh ran 26000 tokens and returned empty content because truncation landed inside the think block, while the same rig at xhigh with a 1500-token budget returned more working code than medium did. Records the standalone mtp-Qwen3.8-27B-Q8_0.gguf sidecar as a second remedy for the Q4_0 blk.64 head, and why it is not taken: load_mtp comes from the type list rather than from a draft path, so the target keeps loading its own head and an external sidecar double-pays. Corrects the Muse Glimmer claim that no bundled template exists. models/templates/muse-glimmer.jinja is a test fixture mirroring the GGUF-embedded template, so not pinning it is deliberate rather than forced. build_system.md: the missing GGML_CUDA_FA_ALL_QUANTS failure is a silent placement of FLASH_ATTN_EXT on the CPU backend, not the GGML_ABORT, because supports_op consults the support query first. Also documents -DGGML_SCHED_MAX_COPIES=1 and what it keeps this repo out of. Stale llama.cpp line references refreshed throughout. The vendor/llama.cpp pointer is left uncommitted: the worktree is on the PR 27742 head, which is not reachable from origin/master, and every pointer this repo has committed so far is a master commit. --- CHANGELOG.md | 13 +++++-- docs/build_system.md | 4 +- docs/model_tuning.md | 73 +++++++++++++++++++++++++----------- presets/models_24GB_VRAM.ini | 1 + 4 files changed, 66 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 911b85c..2975bd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier - [Presets] Pin chat-template-file on the Qwen3.8-27B entry - [Presets] Pin reasoning-effort to medium on all Qwen 3.6 and Bonsai entries +- [Presets] Pin reasoning-effort to xhigh on the Qwen3.8-27B entry - [Documentation] Document why Qwen3.8-27B now pins the vendored chat template -- [Documentation] Document the reasoning-effort steering added by chat template v22 -- [Documentation] Document the tools serialization and think-tag spacing fixes in chat template v22 +- [Documentation] Document the reasoning-effort steering added by chat template v22.1 +- [Documentation] Document the tools serialization and think-tag spacing fixes in chat template v22.1 +- [Documentation] Document why Qwen3.8-27B keeps xhigh reasoning and how reasoning-budget bounds it +- [Documentation] Document the standalone Qwen3.8-27B MTP sidecars as an alternative to a re-quant +- [Documentation] Document the GGML_SCHED_MAX_COPIES=1 CUDA build flag - [Documentation] Document the Qwen3.8-27B projector choice and its quantization source - [Documentation] Document that ctx-size above the GGUF context_length is dead VRAM without override-kv - [Documentation] Document Muse-Glimmer-30B RoPE, swa-full, and context-shift behavior @@ -33,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md - [Documentation] Drop the build-configuration summaries from AGENTS.md that docs/build_system.md already carries - [Documentation] List the remaining examples scripts and their help coverage in AGENTS.md -- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22 +- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22.1 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 - [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V @@ -46,6 +50,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature - [Documentation] Correct the claim that reasoning_effort has no CLI flag - [Documentation] Correct the stated cause of the gguf-py row-group conversion slowdown +- [Documentation] Correct the GGML_CUDA_FA_ALL_QUANTS failure mode from an abort to a silent CPU fallback +- [Documentation] Correct the claim that Muse-Glimmer-30B has no bundled chat template +- [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections - [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow - [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer diff --git a/docs/build_system.md b/docs/build_system.md index 711ee5b..a1c5292 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -28,7 +28,9 @@ sections back are in AGENTS.md "Non-obvious behavior". ## CUDA build flags -- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and the dispatcher returns `BEST_FATTN_KERNEL_NONE` -> `GGML_ABORT` for any other K type or any mismatched K/V pair (`ggml-cuda/fattn.cu:424-446`). The flag pulls in the full `fattn-vec*.cu` set so asymmetric / q5 / q4_1 caches work (the presets use `q5_0` K + `q4_1` V). Costs extra `nvcc` compile time. +- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`, and the failure without it is silent.** Without the flag the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and `ggml_cuda_get_best_fattn_kernel` returns `BEST_FATTN_KERNEL_NONE` for `q4_1` / `q5_0` / `q5_1` (`ggml-cuda/fattn.cu:338-356`) or any mismatched K/V pair (`:442-446`). The `GGML_ABORT` at `fattn.cu:574` is unreachable: `supports_op` consults `ggml_cuda_flash_attn_ext_supported` first (`ggml-cuda/ggml-cuda.cu:5287`), so the scheduler simply places `FLASH_ATTN_EXT` on the **CPU** backend — no error, no log line, a ~20-30x prefill collapse (#27109). Both Muse Glimmer entries use `q5_0` K + `q4_1` V and would hit it. The flag pulls in the full `fattn-vec*.cu` set at the cost of extra `nvcc` compile time. + +- **CUDA builds pass `-DGGML_SCHED_MAX_COPIES=1` (ggml's default is 4).** `sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1` (`vendor/llama.cpp/ggml/src/ggml-backend.cpp:1804`), so pinning it to 1 leaves every multi-copy staging branch dead even when `cparams.pipeline_parallel` is true (`src/llama-context.cpp:428-455`). Single-stream decode gains nothing from the extra copies and they cost ~1.6 GiB on a two-device layer split. It is also what keeps this repo out of #26873, where with pipeline parallelism active one image request permanently costs ~39% prefill for the process lifetime — reproduced on Qwen3.8-27B + mmproj and on Muse Glimmer, with `-ot zzz_never_matches=CUDA0` (which disables pipeline parallelism outright) the only workaround found upstream. ## Build parallelism diff --git a/docs/model_tuning.md b/docs/model_tuning.md index 1fe6fa8..02970dd 100644 --- a/docs/model_tuning.md +++ b/docs/model_tuning.md @@ -36,18 +36,40 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. deliberately keep their GGUF-embedded template; froggeric's README claims compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. -- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry does not.** - v22 added Qwen 3.8's reasoning-effort steering but gates it on nothing — the default - resolves to `xhigh` for *every* model the template serves, injecting a ~45-token - "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. - Qwen 3.6 has no trained notion of the concept, so the entries pin `medium`, the one - level for which v22 emits no instruction text at all. Qwen 3.8 *is* trained on it and - its own template defaults to `xhigh`, so that entry leaves the key unset and inherits - the same default it had before the pin. `--reasoning-effort` writes only a template - kwarg (`common/arg.cpp:3650-3660`), which a request can still override - (`tools/server/server-common.cpp:1296-1303`). Unlike the GGUF-embedded 3.8 template, - v22 never raises on an unknown level — `high` is aliased to `xhigh` and anything else - falls back to it. +- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry pins + `xhigh`.** v22 defaulted the level to `xhigh` for *every* model the template serves, injecting + a ~45-token "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. + v22.1 moved that default to `medium`, the one level that injects no instruction text at all. + Qwen 3.6 has no trained notion of the concept, so its entries pin `medium` — under v22 that + overrode a wrong default, under v22.1 it restates the right one, and either way the pin is what + makes the level independent of the template version. Qwen 3.8 *is* trained on it and Qwen's own + template defaults to `xhigh`, so that entry pins `xhigh` explicitly: leaving the key unset + silently downgraded it to `medium` at the v22.1 bump. Never rely on the template default here. + `--reasoning-effort` writes only a template kwarg (`common/arg.cpp:3650-3660`), which a request + can still override (`tools/server/server-common.cpp:1312-1319`), as can a `<|think_low|>` / + `<|think_medium|>` / `<|think_xhigh|>` tag typed inside a message (new in v22.1, stripped before + rendering). Unlike the GGUF-embedded 3.8 template, the vendored one never raises on an unknown + level — v22.1 maps `high` and `max` to `xhigh`, `minimal` to `low`, `none` to thinking off, and + anything else *down* to `medium` rather than up to `xhigh` as v22 did. + +- **`xhigh` is kept on `Qwen3.8-27B`, and `--reasoning-budget` — not a lower level — is the guard + rail for it.** Qwen publishes no per-level benchmarks; every number on the 27B card is at the + `xhigh` default, and the card warns that in multi-turn agentic tasks lower effort "can also lead + to insufficient analysis, more failures, and repeated retries". The entire mechanism is one + injected sentence: 123 rendered chars at `medium` against 332 at `xhigh` — no token, no sampling + change, no budget. The failure that moved froggeric's default to `medium` is a truncation + artifact rather than a quality result: with a finite `max_tokens` and no budget, `xhigh` ran + 26,000 tokens and returned `content_len = 0` because truncation landed inside ``, while + the same rig at `xhigh` with a 1500-token thinking budget returned 18,512 chars of working code, + more than `medium` produced. There is no rung between the two — `high` aliases to `xhigh`. + `--reasoning-budget N` (`common/arg.cpp:3662-3668`) is live on this entry and deliberately left + unset: the qwen3_coder handler supplies `` and `{"", ""}` + (`common/chat.cpp:1181-1184`), the server forwards them (`server-common.cpp:1360-1366`), and on + exhaustion the sampler masks every logit but the forced `` + (`common/reasoning-budget.cpp:119-131`, `:178-185`) so the model concludes instead of being cut + off, re-arming per thinking block (`:146-161`). Unlike `max_tokens` it counts only tokens inside + the block. Set it if an agentic client that sends its own `max_tokens` starts returning empty + content; a request can override it per call via `reasoning_budget_tokens`. - **`Qwen3.8-27B` pins the template too — v22 removed the reason it used to be the exception.** Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as @@ -63,11 +85,11 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. - v22 handles all three and its `xhigh` instruction text is byte-identical to the - official one, so `reasoning = on` with no `reasoning-effort` key reproduces the + v22.1 handles all three and its `xhigh` instruction text is byte-identical to the + official one, so `reasoning = on` with `reasoning-effort = xhigh` reproduces the pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is selected purely on `` + `` block. The pin additionally brings froggeric's agentic extras (two-tier tool-error escalation, `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). @@ -100,8 +122,14 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. multiplies the recurrent-state buffer by `1 + n_max` — ~150 MiB becomes ~600 MiB at 3. Both `Qwen3.8-27B` and `Qwen3.6-27B` carry `blk.64` (the MTP head) at `Q4_0` in the local IQ4_XS files; a 4-bit MTP head is reported to collapse acceptance to 0% on this - model family, so check the server's acceptance rate before trusting the speedup — - the fix would be a re-quant keeping `blk.64` at `Q5_K` or above, not a preset change. + model family, so check the server's acceptance rate before trusting the speedup. Two + remedies exist: a re-quant keeping `blk.64` at `Q5_K` or above, or — since 2026-08-14 — + pointing `spec-draft-model` at `ggml-org/Qwen3.8-27B-GGUF`'s standalone + `mtp-Qwen3.8-27B-Q8_0.gguf`. Neither is taken. The sidecar is 3.16 GB against the + ~2.65 GiB of headroom measured below, the `Q4_0` sidecar is the same precision as the + embedded head, and `mparams.load_mtp` is set from the *type list* rather than from the + presence of a draft path (`common/common.cpp:1689`, `src/models/qwen35.cpp:42`), so the + target keeps loading its own `blk.64` and an external sidecar double-pays. - **`Qwen3.8-27B` is the only 24 GB Qwen entry on a `Q8_0` projector instead of `BF16`.** 600 MiB rather than 888 MiB of VRAM, and that saving is what keeps `mmproj-offload = true` @@ -313,16 +341,19 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. (`1 + n_accepted / n_verif_steps`, `server-context.cpp:620`), never by `draft acceptance` — that ratio is a percentage of *drafted* tokens and is meaningless for block diffusion, where unaccepted drafts cost nothing. Measured ~3.4 tokens per target pass at short context, ~3.0 at - 4.3k. No template may be pinned: `common/chat.cpp:3276` selects the native Muse Glimmer handler - on `` + `<|eom|>` in the template source, and there is no bundled file to - pin. `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads + 4.3k. The template is deliberately not pinned: `common/chat.cpp:3494` selects the native Muse + Glimmer handler on `` + `<|eom|>` in the template source, and the bundled + `models/templates/muse-glimmer.jinja` is a test fixture added alongside a `chat.cpp` parser fix + (#26879, locked by `tests/test-chat.cpp:5951`) that mirrors the GGUF-embedded template — unlike + gemma-4, where the pin exists to replace genuinely outdated conversions. + `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads `enable_thinking`; reasoning here is the structural ` to=self<|message|>...<|eom|>` channel, gated on `reasoning_format` (`chat.cpp:3141`). The template already defaults to `Reasoning strength: high` (`models/templates/muse-glimmer.jinja:84`), which matches Meta's recommendation, and `--reasoning-effort` reaches it only through the `reasoning_effort` -> `reasoning_strength` alias at `common/jinja/caps.cpp:29-33`. `--reasoning-budget` is inert: the handler sets no `thinking_end_tags` - (`server-common.cpp:1343`). + (`server-common.cpp:1360`). - **Compute buffers dominate this entry's headroom and cannot be derived from KV arithmetic.** Measured on a 24463 MiB card at `ctx-size = 524288` / `parallel = 2` / `q5_0` K + `q4_1` V: diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index c314b12..1e1a9f7 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -117,6 +117,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = xhigh kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 From 5e7ef49fc912a0fae97fa410c67ebb42711c6d54 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Thu, 27 Aug 2026 22:36:10 +0200 Subject: [PATCH 16/20] Add Qwen3.8-Flash-Next to the 24 GB tier and document its mmap and no-host requirements Qwen3.8-Flash-Next is arch qwen4exp, not a variant of the qwen35 family, so almost none of the Qwen3.8-27B tuning carries over. 48 blocks split 12 full-attention layers with QSA block-sparse attention over an indexer cache and 36 gated-delta-net layers, 512 experts with 10 used, native context 262144, qwen3vl_merger projector. The local IQ4_XS is 90.635 GiB and decomposes as 26.822 GiB per_layer_token_embd, 60.938 GiB routed experts at 1.270 GiB per layer, and 2.542 GiB of everything else. This is the one entry in the tier that must use load-mode = mmap. The 26.822 GiB n-gram hash table is created with TENSOR_READ_LAZY and the loader gates that flag on use_mmap, which every non-mmap load mode clears. A token gathers 16 of the table's 320 million rows, so mmap keeps the resident working set in the hundreds of MiB while dio reads and holds all 26.822 GiB for nothing. no-host = true is part of the same mechanism rather than an independent choice: without it the CPU buffer list is prepended with CUDA_Host, the chosen buffer type then fails the is_default_buft test and the mmap aliasing branch is skipped entirely, so every CPU-resident tensor goes through cudaMallocHost and the load dies as the same misleading CUDA OOM that DeepSeek hit. The table itself is unoffloadable: PER_LAYER_TOKEN_EMBD is classified LAYER_INPUT and input tensors are pinned to the CPU list regardless of -ngl, so -ngl is not the lever and the only question is how many expert layers survive beside the KV cache. fit handles that at sub-layer granularity and its overflow pattern matches this arch's _exps names, but it is blind twice over: it assumes host memory is unlimited and never consults the CPU slot once a GPU is present, and it measures with load mode NONE so its host figure counts the table non-lazily. Neither matters on this box, but they are why the split cannot be checked against fit's own host accounting. cache-type-k stays at q8_0 because it also types the QSA indexer cache, whose K is what top_k ranks blocks on, so cheapening it changes which tokens are attended rather than only their values. Two thirds of the indexer allocation is dead weight: hybrid_idx sets n_embd_head_k_full but not n_embd_head_v_full, so is_mla is false, a 256-wide V is allocated, and the graph only ever calls cpy_k and get_k. Dropping cache-type-v to q4_0 would recover 1152 MiB at 262144 with the cost paid only by the 12 real attention layers; it is left for a later measured pass. Unlike deepseek4 the two types may legally differ here. There is no MTP head, so ngram-mod is the only speculative type. IMRoPE forces n_pos_per_embd to 4, which makes get_can_shift false, so the server force-disables context shift and cache reuse with two warnings that are expected on startup; speculative rollback then goes through checkpoints, which is what makes ctx-checkpoints load-bearing rather than an optimisation. no-mmproj-offload = true because fit measures the language model alone and CLIP allocates after the split is committed, the exact silent-OOM window presets.md warns about. The chat template pin is the same file as Qwen3.8-27B's, byte for byte at 8952 bytes, so the three template defects and their fix carry over unchanged, along with reasoning-effort = xhigh. The vendored template renders the vision placeholders, so pinning it costs nothing on the image path. The vendor/llama.cpp pointer is bumped to b10660, the merge commit of upstream 27742 and reachable from origin/master, which is what makes this preset loadable at all. presets/README.md carried two stale claims from the v22.1 bump: that the vendored template defaults to xhigh and that Qwen3.8-27B leaves reasoning-effort unset. Both are corrected. Not yet verified against a running server: the build was still linking when this landed. The weight decomposition and the KV arithmetic are exact, but fit's chosen split, the QSA compute buffer at 262144, the per-checkpoint size and throughput are all unmeasured, so fit-target = 3072 and ctx-checkpoints = 32 are carried over from the DeepSeek entry rather than fitted to this model. --- AGENTS.md | 6 +- CHANGELOG.md | 12 ++++ docs/model_tuning.md | 110 +++++++++++++++++++++++++++++++++++ docs/presets.md | 8 +++ presets/README.md | 14 ++--- presets/models_24GB_VRAM.ini | 33 +++++++++++ vendor/llama.cpp | 2 +- 7 files changed, 174 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fb329c0..8adc264 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,13 +45,13 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini Prohibitions that cause a silent OOM, silent corruption, or a startup abort. Each is stated here without rationale so it is always in context; read the linked section before acting on one. These lines are a deliberate projection of `docs/` — when a trap changes, both move together. -- Never pair `direct-io` with `no-mmap`; use `load-mode = dio`. `docs/presets.md` -> *load-mode* +- Never pair `direct-io` with `no-mmap`; use `load-mode = dio` — except on `Qwen3.8-Flash-Next`, which must keep `load-mode = mmap`. `docs/presets.md` -> *load-mode* - Never set `mmproj-offload = true` on a tier where LLM + KV already saturate VRAM. `docs/presets.md` -> *mmproj-offload* - Never set `swa-full` on a DeepSeek-V4-Flash or Muse Glimmer entry. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Muse Glimmer* - Never set `context-shift` on a Muse Glimmer entry; it is silent corruption, not a refusal. `docs/model_tuning.md` -> *Muse Glimmer* - Never add RoPE scaling to a Muse Glimmer entry. `docs/model_tuning.md` -> *Muse Glimmer* -- `no-host = true` is mandatory on the DeepSeek entry; without it the load fails as a misleading CUDA OOM. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* -- Keep `fit = on` on the DeepSeek entry; never add `n-cpu-moe`/`-ot`, and never set `n-gpu-layers` to anything but `-1`. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* +- `no-host = true` is mandatory on the DeepSeek and `Qwen3.8-Flash-Next` entries, and on any entry that pushes tens of GiB of experts to CPU; without it the load fails as a misleading CUDA OOM. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Qwen3.8-Flash-Next* +- Keep `fit = on` on the DeepSeek and `Qwen3.8-Flash-Next` entries; never add `n-cpu-moe`/`-ot`, and never set `n-gpu-layers` to anything but `-1` — fit then silently no-ops. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Qwen3.8-Flash-Next* - `cache-type-k` and `cache-type-v` must be identical on `deepseek4`; differing values are startup-fatal. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* - Never set `image-min-tokens` on a gemma-4 entry; it is a `qwen3vl_merger` key only. `docs/model_tuning.md` -> *Qwen 3.6 and 3.8* - Never drop a `chat-template-file` pin; it replaces the GGUF-embedded template and is not redundant with `jinja = true`. `docs/model_tuning.md` diff --git a/CHANGELOG.md b/CHANGELOG.md index 2975bd9..f648ee1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document the local patch mechanism in the build system reference - [Documentation] Document the gguf-py row-group fix and how it was verified - [Documentation] Document that the gguf-py row-group fix is a throughput fix and not the PLE table OOM fix +- [Presets] Add Qwen3.8-Flash-Next IQ4_XS entry to the 24 GB tier +- [Documentation] Document the Qwen3.8-Flash-Next per_layer_token_embd table and its lazy read path +- [Documentation] Document why the Qwen3.8-Flash-Next entry needs load-mode = mmap instead of dio +- [Documentation] Document why no-host keeps the Qwen3.8-Flash-Next mmap aliasing path reachable +- [Documentation] Document the Qwen3.8-Flash-Next weight split and the two fit blind spots +- [Documentation] Document the Qwen3.8-Flash-Next QSA indexer cache and its unused V allocation +- [Documentation] Document that Qwen3.8-Flash-Next has no MTP head and cannot shift context or reuse cache +- [Documentation] Document the Qwen3.8-Flash-Next recurrent state cost and why parallel is the cheap knob ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md @@ -41,6 +49,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 - [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V +- [Vendor] Bump llama.cpp submodule to b10660 for Qwen3.8-Flash-Next support #27742 +- [Documentation] Extend the no-host and fit traps in AGENTS.md to Qwen3.8-Flash-Next ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier @@ -53,6 +63,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Correct the GGML_CUDA_FA_ALL_QUANTS failure mode from an abort to a silent CPU fallback - [Documentation] Correct the claim that Muse-Glimmer-30B has no bundled chat template - [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections +- [Documentation] Correct the presets README claim that Qwen3.8-27B leaves reasoning-effort unset +- [Documentation] Correct the presets README claim that the vendored chat template defaults to xhigh - [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow - [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer diff --git a/docs/model_tuning.md b/docs/model_tuning.md index 02970dd..95203af 100644 --- a/docs/model_tuning.md +++ b/docs/model_tuning.md @@ -153,6 +153,116 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. MTP head is not — `mtp.layers.0`'s attention and MLP projections are FP8 there, so any re-quant raising `blk.64` above 4-bit must also come from the BF16 repo. +## Qwen3.8-Flash-Next + +Arch `qwen4exp` (upstream #27742, merged at `b10660`), a separate architecture from the `qwen35` +family above and tuned on different grounds. 48 blocks: 12 full-attention layers carrying QSA +block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 experts with 10 used, +`context_length = 262144`, `qwen3vl_merger` projector. + +- **`load-mode = mmap` and `no-host = true` are one mechanism, not two independent choices.** The + 26.822 GiB `per_layer_token_embd` n-gram hash table (`Q4_0`, 160 x 320,001,536) is created with + `TENSOR_READ_LAZY` (`src/models/qwen4exp.cpp:139-140`), and the loader gates that flag on + `use_mmap` (`src/llama-model-loader.cpp:1290`), which only `mmap` / `mmap+mlock` / `auto` set + (`:559`). The `auto` threshold is 4 GiB (`:1292`), so the table qualifies without pinning + `tensor-read-lazy`. A token gathers `ple_n_heads = (ngram_size - 1) * heads_per_ngram` = 16 rows + (`src/models/qwen4exp.cpp:64`, gather at `:1106-1112`), so a session touches a vanishing fraction + of the 320 M rows and mmap — which aliases the file rather than copying it + (`src/llama-model-loader.cpp:1578-1602`) — keeps the resident working set in the hundreds of MiB. + `dio` reads and holds all 26.822 GiB for nothing; this is the one entry in the tier that must not + use it. `no-host` is what keeps that path reachable: without it `make_cpu_buft_list` prepends + `CUDA_Host` to the CPU list (`src/llama-model.cpp:1047-1049`), the chosen buft then fails the + `is_default_buft` test at `:1715`, the mmap-aliasing branch at `:1718` is skipped entirely, and + every CPU-resident tensor — tens of GiB of experts plus the 26.822 GiB table — goes through + `cudaMallocHost`, which is the misleading CUDA OOM documented for DeepSeek below. `mmap+mlock` is + the other wrong answer: it forces the whole table resident + (`src/llama-model-loader.cpp:1595-1598`). + +- **The table can never be offloaded, so `-ngl` is not the lever and the entire budget question is + expert layers.** `LLM_TENSOR_PER_LAYER_TOKEN_EMBD` is classified `LLM_TENSOR_LAYER_INPUT` + (`src/llama-arch.cpp:887`) and `src/llama-model.cpp:1482-1483` pins every input tensor to + `cpu_buft_list` regardless of `-ngl` ("there is very little benefit to offloading the input + layer"); only an explicit `-ot` could move it. Composition of the local IQ4_XS (90.635 GiB, 1224 + tensors): 26.822 GiB PLE table plus 0.333 GiB `token_embd` on the CPU by construction, 60.938 GiB + of routed experts (1.270 GiB per layer x 48) for `fit` to place, and 2.542 GiB of everything else + on the GPU. The GPU-side floor is therefore small, and how many of the 48 expert layers survive + beside the KV cache is the only thing that moves throughput. + +- **`fit = on` with an explicit `ctx-size`, and fit is blind in two ways here.** Fit reduces context + only when the user left it unset (`common/fit.cpp:197` `n_ctx_auto = n_ctx == 0`, used at `:392`), + so `ctx-size = 262144` is honoured and step 3 spends the remainder on expert fractions — its + overflow pattern `blk\.N\.ffn_(up|down|gate_up|gate)_(ch|)exps` (`common/fit.cpp:522`) matches + exactly what this arch names them (`src/models/qwen4exp.cpp:202-203`). Blind spot one: fit assumes + host memory is unlimited (`common/fit.h:24`) and never consults the CPU slot once a GPU is present + (`common/fit.cpp:331-347`). Blind spot two: it measures with `load_mode = LLAMA_LOAD_MODE_NONE` + (`common/fit.cpp:57`), so lazy read is off during measurement and its host figure counts the full + PLE table — a number the real run never produces. Neither matters on a 191 GiB box, but they are + why the split cannot be sanity-checked from fit's own host accounting. Setting `n-gpu-layers` to + anything but `-1`, or any `-ot` / `--cpu-moe` / `--n-cpu-moe`, throws inside fit + (`common/fit.cpp:462-464`, `:483-485`) and is downgraded to a warning at `:893-895`: fit silently + does nothing and the model no longer fits at all. + +- **`cache-type-k` also types the QSA indexer cache, which is why K stays at `q8_0`.** + `src/llama-model.cpp:2506-2507` hands `params.type_k` / `type_v` to `llama_memory_hybrid_idx`, + which forwards them unchanged to the indexer cache (`src/llama-memory-hybrid-idx.cpp:56`). + Indexer K is what `ggml_top_k` ranks blocks on (`src/models/qwen4exp.cpp:599-601`), so cheapening + it degrades *which* tokens are attended, not just their values. Per-token cost at + `kv-unified = true`: 12 attention layers x 1024 elements (`n_head_kv = 2` x `head = 256`, K and V) + plus 12 indexer layers x 384 elements = 17,952 B at `q8_0`, i.e. 4,488 MiB at 262144. Two thirds + of the indexer share is dead — `src/llama-memory-hybrid-idx.cpp:50-51` sets `n_embd_head_k_full` + but not `n_embd_head_v_full`, so `is_mla()` is false, `src/llama-kv-cache.cpp:232-235` allocates a + 256-wide V, and the graph only ever calls `cpy_k` / `get_k` (`src/models/qwen4exp.cpp:530`, + `:533`). Dropping `cache-type-v` to `q4_0` would recover 1,152 MiB at this context with the + quality cost paid only by the 12 real attention layers; worth trying, not taken. Unlike + `deepseek4` the two types may legally differ — the equality guard at + `src/llama-context.cpp:3592-3595` fires only for `is_mla()` or `LLM_ARCH_DEEPSEEK4`. + +- **Recurrent state is 112.219 MiB per sequence and independent of `ctx-size`, so `parallel` is the + cheap knob and context the expensive one.** 36 gated-delta-net layers at + `n_embd_r = 3 x 10240` and `n_embd_s = 128 x 6144` elements (`src/llama-hparams.cpp:204`, `:232`), + both hardcoded `GGML_TYPE_F32` (`src/llama-model.cpp:2513-2514`) so no cache type shrinks them, + and one row per sequence because `qwen4exp` is absent from `llm_arch_supports_rs_rollback` + (`src/llama-arch.cpp:1099-1113`) and `n_rs_seq` is clamped to 0 at + `src/llama-context.cpp:105-108`. The entry still keeps `parallel = 1`: with `kv-unified = true` + `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`), so extra slots share one pool rather than + extending it. + +- **No MTP head, and context shift and cache-reuse are structurally impossible — which is what makes + `ctx-checkpoints` load-bearing.** `conversion/qwen4exp.py:28-30` drops the MTP block ("a separate + draft head; vLLM drops it too"), so the GGUF carries no `nextn` tensors and `spec-type = + draft-mtp` fails at `src/llama-context.cpp:3637-3642`; `ngram-mod` is the only speculative type + available. `get_can_shift()` is false because IMRoPE gives `n_pos_per_embd() == 4` + (`src/llama-kv-cache.cpp:1194-1196`, rope type at `src/llama-model.cpp:2951-2955`), so the server + force-disables context shift *and* cache-reuse with two warnings at + `tools/server/server-context.cpp:1185-1195` — both are expected on startup, not a + misconfiguration. Speculative rollback then goes through checkpoints + (`tools/server/server-context.cpp:1224-1226`), so `ctx-checkpoints = 32` is required for + `ngram-mod` to be useful rather than being an optimisation. `swa-full` is inert: `swa_type` is + `NONE`, which is why the model takes the `hybrid_idx` path at `src/llama-model.cpp:2502` at all, + and the server clears the flag at `:1197-1202`. + +- **The template pin is the same file as `Qwen3.8-27B`'s, byte for byte.** The embedded template is + 8952 bytes with sha256 `c3cf9e34abf4f9e3...` — identical to Qwen3.8-27B's — so the three defects + documented above (tool-call `arguments` arriving as a JSON string, a duplicate blank + `\n\n` from history, `raise_exception` on `high` / `minimal` / `max`) apply + unchanged, and so does the fix. `reasoning-effort = xhigh` for the same reason; `--reasoning-budget` + remains the guard rail rather than a lower level. Pinning costs no vision: the vendored template + renders `<|vision_start|><|image_pad|><|vision_end|>` + (`vendor/Qwen-Fixed-Chat-Templates/chat_template.jinja:67-88`). The projector is + `qwen3vl_merger`, so `image-min-tokens = 1024` applies as it does to the other Qwen-VL entries. + The sampler block restates the GGUF's own `general.sampling.*` (`temp 1.0`, `top-p 0.95`, + `top-k 20`), matching Qwen3.8-27B. + +- **`no-mmproj-offload = true`, because CLIP is exactly what `fit` cannot see.** Fit measures the + language model alone and commits the expert split before `mtmd` loads the projector, so a + `mmproj-offload = true` here lands in the silent-OOM window described in `docs/presets.md` -> + *mmproj-offload* with no margin left to absorb it — the 588 MiB of `Q8_0` weights plus ~310 MiB of + CLIP compute would have to be reserved by hand through `fit-target`. `fit-target = 3072` is the + same WDDM-plus-untracked-CUDA-scratch margin as the DeepSeek entry on the same card, for the same + reasons. `cache-ram = 32768` rather than the 51200 used by the small entries: a full-context + prompt state is ~4.6 GiB here (4,488 MiB KV plus the recurrent rows), and the host is already + backing most of a 90.635 GiB file through the page cache plus a PLE working set that only grows. + ## gemma-4 - **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** diff --git a/docs/presets.md b/docs/presets.md index 2866cf3..cfb08b8 100644 --- a/docs/presets.md +++ b/docs/presets.md @@ -36,6 +36,14 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini values are `none`, `mmap`, `mlock`, `mmap+mlock`, `dio` (`arg.cpp:2615-2619`) — anything else throws at startup. +- **`Qwen3.8-Flash-Next` is the one entry that must use `load-mode = mmap` instead.** Its 26.8 GiB + `per_layer_token_embd` n-gram hash table is created with `TENSOR_READ_LAZY` + (`src/models/qwen4exp.cpp:139-140`) and the loader gates that flag on `use_mmap` + (`src/llama-model-loader.cpp:1290`), which every non-mmap `load-mode` clears (`:559`). A session + touches 8 of the table's 320 million rows per token, so under `mmap` the resident working set + stays in the hundreds of MiB while `dio` reads and holds all 26.8 GiB. `no-host = true` is part + of the same mechanism, not an independent choice — see `docs/model_tuning.md` -> *Qwen3.8-Flash-Next*. + - **`load-mode = dio` does not enable DirectIO on Windows — it only disables mmap.** The Win32 `llama_file::impl` ctor takes `use_direct_io` as `[[maybe_unused]]` and just calls `ggml_fopen` (`vendor/llama.cpp/src/llama-mmap.cpp:86-95`); `FILE_FLAG_NO_BUFFERING` is never set and `read_alignment()` stays 1 (`:391`), so the loader's async staging buffers are 4 x 1 MiB of pinned host memory instead of the 4 x 64 MiB the aligned path would use (`src/llama-model-loader.cpp:1418`, `:1427`). `has_direct_io()` nevertheless returns a hardcoded `true` on Windows (`:173-175`). Net effect of `dio` on this platform: buffered reads, no mmap, and zero VRAM cost — it is never implicated in a CUDA OOM. Keep the key for the deprecation-warning reason documented above, but do not reason about page-cache behaviour from it. ## mmproj-offload diff --git a/presets/README.md b/presets/README.md index e9fd11b..49de12b 100644 --- a/presets/README.md +++ b/presets/README.md @@ -31,7 +31,7 @@ llama-server --models-dir D:\AI\LLM\gguf --models-preset presets\models_16GB_8GB > — they are parent-server settings that the server manages internally and cannot be set via preset. > [!NOTE] -The presets `models_16GB_VRAM.ini`, `models_24GB_VRAM.ini`, and `models_16GB_8GB_VRAM.ini` are each tuned for its VRAM budget (context size, KV quantisation, and MoE offload differ). Copy one as a starting point for other hardware. Only `models_16GB_8GB_VRAM.ini` pins GPUs — the other two leave `split-mode`/`tensor-split` unset, so on a multi-GPU host they spread across every visible CUDA device and may exceed the budget named in the file. Pin with `CUDA_VISIBLE_DEVICES` before launching; its indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and does **not** match `nvidia-smi` ordering, so a GPU UUID is the unambiguous choice. +The presets `models_16GB_VRAM.ini`, `models_24GB_VRAM.ini`, and `models_16GB_8GB_VRAM.ini` are each tuned for its VRAM budget (context size, KV quantisation, and MoE offload differ). Copy one as a starting point for other hardware. Only `models_16GB_8GB_VRAM.ini` pins GPUs — the other two leave `split-mode`/`tensor-split` unset, so on a multi-GPU host they spread across every visible CUDA device and may exceed the budget named in the file. Pin with `CUDA_VISIBLE_DEVICES` before launching; its indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and does **not** match `nvidia-smi` ordering, so a GPU UUID is the unambiguous choice. Every entry uses `load-mode = dio` except `Qwen3.8-Flash-Next`, which needs `mmap` so that its 26.8 GiB n-gram embedding table can be read on demand — do not normalise that one away. > [!IMPORTANT] > **`models_16GB_8GB_VRAM.ini` (dual-GPU: one GPU with 16 GB VRAM + one with 8 GB VRAM).** @@ -78,12 +78,12 @@ The section header (e.g. `[gemma-4-31B-it.IQ4_XS.gguf]`) is the model name clien > If you cloned without `--recurse-submodules`, run `git submodule update --init` > first — otherwise startup fails with a missing-file error. > -> The Qwen 3.6 and Bonsai entries additionally set `reasoning-effort = medium`. The vendored -> template defaults to `xhigh`, which prepends a "Reasoning effort is set to xhigh..." -> paragraph to the system prompt; `medium` is the one level that emits no instruction text, -> and Qwen 3.6 has no trained notion of the concept. `Qwen3.8-27B` leaves the key unset -> because it *is* trained on it and `xhigh` is its own template's default too. Clients can -> still override per request via the OpenAI `reasoning_effort` field. +> The Qwen 3.6 and Bonsai entries additionally set `reasoning-effort = medium`, the `Qwen3.8-*` +> entries `xhigh`. `medium` is the one level that injects no instruction text into the system +> prompt, and Qwen 3.6 has no trained notion of the concept; Qwen 3.8 *is* trained on it and +> `xhigh` is what Qwen's own template defaults to. Both are pinned rather than left unset because +> the vendored template's default has moved before (v22 shipped `xhigh`, v22.1 `medium`). Clients +> can still override per request via the OpenAI `reasoning_effort` field. > > All `gemma-4-*` entries set `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja` — > the official Google template bundled with llama.cpp itself, kept in lock-step with diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 1e1a9f7..a90bfae 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -168,6 +168,39 @@ spec-ngram-mod-n-max = 64 load-mode = dio fit = off +[Qwen3.8-Flash-Next.IQ4_XS.gguf] +alias = Qwen3.8-Flash-Next.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\Qwen3.8-Flash-Next.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\mmproj.Qwen3.8-Flash-Next.Q8_0.gguf +no-mmproj-offload = true +image-min-tokens = 1024 +ctx-size = 262144 +parallel = 1 +n-gpu-layers = -1 +cache-type-k = q8_0 +cache-type-v = q8_0 +flash-attn = true +reasoning = on +jinja = true +chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = xhigh +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 32768 +min-p = 0.0 +top-p = 0.95 +top-k = 20 +temp = 1.0 +presence-penalty = 0 +spec-type = ngram-mod +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = mmap +no-host = true +fit = on +fit-target = 3072 + [Muse-Glimmer-30B.IQ4_XS.gguf] alias = Muse-Glimmer-30B.IQ4_XS.gguf model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ4_XS.gguf diff --git a/vendor/llama.cpp b/vendor/llama.cpp index a4a4c51..6c84c7d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit a4a4c51f3d40e086b59b73b631b5c43c8fbf4504 +Subproject commit 6c84c7d5d8833c6e0df69628f75a0f599797934e From 258eb3c73d6413e8016729ecc9a70f34b72ab45d Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Thu, 27 Aug 2026 23:25:51 +0200 Subject: [PATCH 17/20] Raise Qwen3.8-Flash-Next to parallel 4 and record the measured 24 GB ceiling The entry loaded and ran on the first try, so this replaces the arithmetic-only figures from the previous commit with measured ones: on a 24463 MiB card at ctx-size 262144, parallel 1, q8_0 K and V and CLIP on the CPU, the child sits at 20174 MiB used with 3964 MiB free and generates 19.87 t/s at short context. The free figure tracks fit-target 3072 plus roughly 0.9 GiB of slack, so the margin is neither wrong nor obviously over-provisioned and stays as it is. parallel goes to 4 because it is nearly free here. kv-unified makes n_ctx_seq equal n_ctx, so the four slots share the 262144-cell pool rather than each being handed one; the KV cache does not change size and the only VRAM the extra slots add is three more recurrent rows, 336.7 MiB against 3964 free. A single long conversation can still occupy the whole pool. ctx-checkpoints drops from 32 to 8 in the same breath, because raising parallel without that would have quietly reserved 14.3 GiB of host RAM. A checkpoint on this arch is not the cheap thing it is on DeepSeek: checkpoints are written with PARTIAL_ONLY, that flag skips the attention cache and the indexer cache, and what is left is the entire 112.219 MiB recurrent state rather than a 14.5 MiB window. Checkpoints are also per slot, so the budget is parallel times ctx-checkpoints times 112 MiB. 8 holds it at the same ~3.6 GiB that parallel 1 with 32 checkpoints would have cost. A 1M cell pool is not reachable on this card and the reason is arithmetic rather than tuning. Going from 262144 to 1048576 takes the KV cache from 4488 to 17952 MiB at q8_0, plus 13464, against 3964 MiB free and at most ~9 GiB recoverable by moving every remaining expert layer to the CPU, and the QSA bias tensors are shaped [n_kv, n_tokens] so the compute buffer grows fourfold on top of that. Even a config that squeezed in would be a throughput regression, since zero expert layers on the GPU is strictly worse than the current split. parallel is not a way around it either: under kv-unified four slots of 262144 is 1048576 cells at identical cost. The only route to a 1M pool would be q4_0 K and V, and cache-type-k is the one value that should not drop because it also types the indexer whose K ranks attention blocks. Also records that a large pool needs no override-kv on this model, unlike Muse Glimmer: n_ctx_train is already 262144, so the server caps each slot there and the capping warning is expected. The server-context line references for that cap were stale in presets.md and are refreshed. --- CHANGELOG.md | 5 +++++ docs/model_tuning.md | 42 +++++++++++++++++++++++++++++++++--- docs/presets.md | 4 ++-- presets/models_24GB_VRAM.ini | 4 ++-- 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f648ee1..00ca9ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V - [Vendor] Bump llama.cpp submodule to b10660 for Qwen3.8-Flash-Next support #27742 - [Documentation] Extend the no-host and fit traps in AGENTS.md to Qwen3.8-Flash-Next +- [Presets] Raise Qwen3.8-Flash-Next parallel to 4 and lower ctx-checkpoints to 8 +- [Documentation] Document the measured Qwen3.8-Flash-Next 24 GB VRAM and throughput figures +- [Documentation] Document that a Qwen3.8-Flash-Next context checkpoint is the whole recurrent state +- [Documentation] Document why a 1M cell pool does not fit Qwen3.8-Flash-Next on 24 GB ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier @@ -65,6 +69,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections - [Documentation] Correct the presets README claim that Qwen3.8-27B leaves reasoning-effort unset - [Documentation] Correct the presets README claim that the vendored chat template defaults to xhigh +- [Documentation] Refresh the stale server-context line references for the n_ctx_train slot cap - [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow - [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer diff --git a/docs/model_tuning.md b/docs/model_tuning.md index 95203af..aa41159 100644 --- a/docs/model_tuning.md +++ b/docs/model_tuning.md @@ -223,9 +223,45 @@ block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 exp both hardcoded `GGML_TYPE_F32` (`src/llama-model.cpp:2513-2514`) so no cache type shrinks them, and one row per sequence because `qwen4exp` is absent from `llm_arch_supports_rs_rollback` (`src/llama-arch.cpp:1099-1113`) and `n_rs_seq` is clamped to 0 at - `src/llama-context.cpp:105-108`. The entry still keeps `parallel = 1`: with `kv-unified = true` - `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`), so extra slots share one pool rather than - extending it. + `src/llama-context.cpp:105-108`. The entry runs `parallel = 4` on the strength of that: with + `kv-unified = true` `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`), so four slots *share* + the 262144-cell pool rather than each being given one, the KV cost is unchanged, and the only + VRAM the extra slots add is three more recurrent rows — 336.7 MiB. A single long conversation can + still occupy the whole pool. + +- **A context checkpoint here is the entire recurrent state, ~112 MiB, and checkpoints are per + slot — which is why `ctx-checkpoints` is 8 and not 32.** Checkpoints are written with + `LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY`, and that flag skips both the attention cache + (`src/llama-memory-hybrid.cpp:191-192`) and the indexer cache + (`src/llama-memory-hybrid-idx.cpp:204-206`), leaving only + `llama_memory_recurrent::state_write` (`src/llama-memory-hybrid.cpp:194`). So the blob is the full + 112.219 MiB rather than the 14.5 MiB a DSV4 checkpoint costs, and because + `slot.prompt.checkpoints` is per slot (`tools/server/server-context.cpp:2283`) the host budget is + `parallel x ctx-checkpoints x 112 MiB`. At `parallel = 4`, 32 checkpoints would reserve up to + 14.3 GiB of host RAM; 8 holds it at the ~3.6 GiB that `parallel = 1` with 32 would have cost. + Raising `parallel` again means lowering this in step. + +- **Measured on a 24463 MiB card at `ctx-size = 262144` / `parallel = 1` / `q8_0` K + V, CLIP on + CPU: 20174 MiB used, 3964 MiB free, 19.87 t/s tg at short context.** The free figure tracks + `fit-target = 3072` plus ~0.9 GiB of slack, so the margin is doing its job and is not obviously + over-provisioned. Throughput is expert-traffic bound, not attention bound: every token reads + 10 of 512 experts across all 48 layers, ~26.1 MiB per layer at IQ4_XS, so the ~1.25 GiB per token + that is not resident on the GPU is what sets the rate. That is the currency `ctx-size` is spent + in — 1,300 MiB of KV is one expert layer is roughly 2 % of tg. + +- **A 1,048,576-cell pool does not fit, and the ceiling is arithmetic rather than a tuning + question.** Going from 262144 to 1048576 takes the KV cache from 4,488 to 17,952 MiB at `q8_0`, + `+13,464 MiB`, against 3,964 MiB measured free plus at most ~9 GiB recoverable by moving every + remaining expert layer to the CPU — and the QSA bias tensors are sized `[n_kv, n_tokens]` + (`src/llama-memory-hybrid-idx.h:132-138`), so the compute buffer grows by the same factor of four + on top. Even if it squeezed in it would be a regression, because zero expert layers on the GPU is + strictly slower than the current split. `parallel` is not a way around it: under `kv-unified` the + slots share one pool, so "four slots of 262144" *is* 1,048,576 cells at identical cost. The only + route to a 1 M pool on 24 GB is `q4_0` K and V, which halves it to 9,504 MiB, and `cache-type-k` + is precisely the value that should not drop because it types the indexer. Note that a large pool + needs no `override-kv`: `n_ctx_train` is 262144, so the server caps each slot there + (`tools/server/server-context.cpp:1209-1214`, applied at `:1274`) and the `- capping` line is + expected rather than a misconfiguration. - **No MTP head, and context shift and cache-reuse are structurally impossible — which is what makes `ctx-checkpoints` load-bearing.** `conversion/qwen4exp.py:28-30` drops the MTP block ("a separate diff --git a/docs/presets.md b/docs/presets.md index cfb08b8..faa0e3e 100644 --- a/docs/presets.md +++ b/docs/presets.md @@ -57,8 +57,8 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini - **`ctx-size` above the GGUF's `context_length` is dead VRAM unless `override-kv` lifts it too.** `llama-context.cpp:131` never clamps `n_ctx`, so the KV cache really is allocated at the requested size — but the server then caps every slot at `n_ctx_train` - (`tools/server/server-context.cpp:1201-1203`, applied as `slot.n_ctx = n_ctx_slot` at `:1255`) - and rejects any larger request outright at `:3100` / `:3111`. Both Muse Glimmer GGUFs ship + (`tools/server/server-context.cpp:1209-1214`, applied as `slot.n_ctx = n_ctx_slot` at `:1274`) + and rejects any larger request outright. Both Muse Glimmer GGUFs ship `muse-glimmer.context_length = 131072`, so the 24 GB entry's former `ctx-size = 262144` allocated 262144 cells while no request could exceed 131072 — ~884 MiB of unreachable VRAM. `override-kv = muse-glimmer.context_length=int:262144` raises `n_ctx_train` and is the only diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index a90bfae..98e91e6 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -175,7 +175,7 @@ mmproj = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\mmproj.Qwen3.8-Flash-Next.Q8_0.gguf no-mmproj-offload = true image-min-tokens = 1024 ctx-size = 262144 -parallel = 1 +parallel = 4 n-gpu-layers = -1 cache-type-k = q8_0 cache-type-v = q8_0 @@ -185,7 +185,7 @@ jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja reasoning-effort = xhigh kv-unified = true -ctx-checkpoints = 32 +ctx-checkpoints = 8 cache-ram = 32768 min-p = 0.0 top-p = 0.95 From 8dc79b1d2a264cdb73acb1dd9e80f7c01f1caea0 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Fri, 28 Aug 2026 17:13:20 +0200 Subject: [PATCH 18/20] Raise Qwen3.8-Flash-Next to a 524288 pool, bump the chat template to v22.4, and restate 1.41.0 as a release ctx-size goes to 524288 with parallel 2. Under kv-unified n_ctx_seq equals n_ctx while the server still caps each slot at n_ctx_train, so this is two slots that can each independently reach the model's native 262144, rather than four slots contending for one 262144 pool. It needs no override-kv because n_ctx_train is already 262144. It is not free, and the cost is invisible in "VRAM used" because fit always fills to the fit-target margin. llama-fit-params gives the fixed cost per n_ctx_seq directly: 1234 context and 573 compute at 65536, 4600 and 1821 at 262144, 9088 and 3497 at 524288, 18064 and 6829 at 1048576, plus 112 MiB per sequence beyond the first. So the shipped config's fixed cost is 12697 MiB against 6758 for 262144 with parallel 4, and that ~5900 MiB comes straight out of GPU-resident expert layers, roughly halving them from ~8.3 of 48 to ~4.4. Measured: 19950 MiB used, 4513 free, 14.90 t/s on a 400-token prose completion. The docs record that a single conversation is capped at 262144 either way, so on a single-user workload the larger pool is paid for and unused. The chat template submodule moves from 9f14778 to e649070. The pointer had never left v22 despite the CHANGELOG claiming a v22.1 bump and model_tuning.md describing v22.1 semantics that were not checked out; upstream had reached v22.4. Because every entry pins reasoning-effort explicitly, the default-effort move to medium cannot reach us: rendering both versions across plain chat, system prompt, thinking-off, multi-turn-with-thinking, vision and both tool-argument wire formats is byte-identical wherever the level is pinned. The single behavioural delta is an empty think block before a historical tool call whose assistant message carried no reasoning, which is deliberate upstream token alignment with what Qwen itself generates, and is suppressed by round-tripping reasoning_content, thinking, message.reasoning or an inline think block. Verified end-to-end through minja rather than only in jinja2: a tool call is emitted and parsed, and the follow-up turn returns finish_reason stop with correct synthesis for dict arguments, JSON-string arguments, and with reasoning present. 1.41.0 is unreleased, so the CHANGELOG and docs now state what is in it rather than what happened between iterations. The last released template was v19, so the net change is v19 to v22.4 and the intermediate v22 and v22.1 references are gone. Bullets that corrected or revised something introduced within 1.41.0 itself are removed rather than recorded, since no reader ever saw the intermediate state: the parallel 4 step, the gguf-py row-group cause correction, and both presets README corrections. Date set to today. Also documents that the desktop runs on the Intel iGPU, which is why the NVIDIA card reports 23151 of 24462 MiB free and fit-target 3072 is covering CUDA and WDDM scratch rather than a compositor. --- CHANGELOG.md | 36 ++++----- docs/model_tuning.md | 130 ++++++++++++++++++++----------- docs/presets.md | 16 ++++ presets/README.md | 5 +- presets/models_24GB_VRAM.ini | 4 +- vendor/Qwen-Fixed-Chat-Templates | 2 +- 6 files changed, 126 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00ca9ae..0683a53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,16 +5,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.41.0] - 2026-08-15 +## [1.41.0] - 2026-08-28 ### Added - [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier +- [Presets] Add Qwen3.8-Flash-Next IQ4_XS entry to the 24 GB tier - [Presets] Pin chat-template-file on the Qwen3.8-27B entry - [Presets] Pin reasoning-effort to medium on all Qwen 3.6 and Bonsai entries -- [Presets] Pin reasoning-effort to xhigh on the Qwen3.8-27B entry +- [Presets] Pin reasoning-effort to xhigh on the Qwen3.8 entries - [Documentation] Document why Qwen3.8-27B now pins the vendored chat template -- [Documentation] Document the reasoning-effort steering added by chat template v22.1 -- [Documentation] Document the tools serialization and think-tag spacing fixes in chat template v22.1 +- [Documentation] Document the reasoning-effort steering and the default-effort knob in the vendored chat template +- [Documentation] Document the reasoning-effort aliases the vendored template maps for OpenAI, Claude Code, Cursor and Cline +- [Documentation] Document the tools serialization and think-tag spacing fixes in the vendored chat template +- [Documentation] Document the two v19 prompt-format deviations the vendored template corrects +- [Documentation] Document the empty think block the vendored template emits before a tool call that carries no reasoning +- [Documentation] Document that pinning reasoning-effort is what keeps a template bump from moving the level - [Documentation] Document why Qwen3.8-27B keeps xhigh reasoning and how reasoning-budget bounds it - [Documentation] Document the standalone Qwen3.8-27B MTP sidecars as an alternative to a re-quant - [Documentation] Document the GGML_SCHED_MAX_COPIES=1 CUDA build flag @@ -29,9 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Build] Apply patches from ./patches to the llama.cpp submodule after checkout - [Build] Abort the build when a local patch no longer applies - [Documentation] Document the local patch mechanism in the build system reference -- [Documentation] Document the gguf-py row-group fix and how it was verified -- [Documentation] Document that the gguf-py row-group fix is a throughput fix and not the PLE table OOM fix -- [Presets] Add Qwen3.8-Flash-Next IQ4_XS entry to the 24 GB tier +- [Documentation] Document the gguf-py row-group fix, how it was verified, and that it is a throughput fix rather than the PLE table OOM fix - [Documentation] Document the Qwen3.8-Flash-Next per_layer_token_embd table and its lazy read path - [Documentation] Document why the Qwen3.8-Flash-Next entry needs load-mode = mmap instead of dio - [Documentation] Document why no-host keeps the Qwen3.8-Flash-Next mmap aliasing path reachable @@ -39,22 +42,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document the Qwen3.8-Flash-Next QSA indexer cache and its unused V allocation - [Documentation] Document that Qwen3.8-Flash-Next has no MTP head and cannot shift context or reuse cache - [Documentation] Document the Qwen3.8-Flash-Next recurrent state cost and why parallel is the cheap knob +- [Documentation] Document that a Qwen3.8-Flash-Next context checkpoint is the whole recurrent state +- [Documentation] Document the measured Qwen3.8-Flash-Next 24 GB VRAM and throughput figures +- [Documentation] Document the measured Qwen3.8-Flash-Next context and compute cost per n_ctx_seq +- [Documentation] Document why a 1M cell pool does not fit Qwen3.8-Flash-Next on 24 GB +- [Documentation] Document that the fit-target values assume the dGPU does not drive the display ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md - [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md - [Documentation] Drop the build-configuration summaries from AGENTS.md that docs/build_system.md already carries - [Documentation] List the remaining examples scripts and their help coverage in AGENTS.md -- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22.1 +- [Documentation] Extend the no-host and fit traps in AGENTS.md to Qwen3.8-Flash-Next +- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22.4 +- [Vendor] Bump llama.cpp submodule to b10660 for Qwen3.8-Flash-Next support #27742 - [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 - [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 - [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V -- [Vendor] Bump llama.cpp submodule to b10660 for Qwen3.8-Flash-Next support #27742 -- [Documentation] Extend the no-host and fit traps in AGENTS.md to Qwen3.8-Flash-Next -- [Presets] Raise Qwen3.8-Flash-Next parallel to 4 and lower ctx-checkpoints to 8 -- [Documentation] Document the measured Qwen3.8-Flash-Next 24 GB VRAM and throughput figures -- [Documentation] Document that a Qwen3.8-Flash-Next context checkpoint is the whole recurrent state -- [Documentation] Document why a 1M cell pool does not fit Qwen3.8-Flash-Next on 24 GB ### Removed - [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier @@ -63,13 +67,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - [Documentation] Correct the common.cpp line reference for the embedded sampling temperature - [Documentation] Correct the claim that reasoning_effort has no CLI flag -- [Documentation] Correct the stated cause of the gguf-py row-group conversion slowdown - [Documentation] Correct the GGML_CUDA_FA_ALL_QUANTS failure mode from an abort to a silent CPU fallback - [Documentation] Correct the claim that Muse-Glimmer-30B has no bundled chat template - [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections -- [Documentation] Correct the presets README claim that Qwen3.8-27B leaves reasoning-effort unset -- [Documentation] Correct the presets README claim that the vendored chat template defaults to xhigh -- [Documentation] Refresh the stale server-context line references for the n_ctx_train slot cap - [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow - [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer diff --git a/docs/model_tuning.md b/docs/model_tuning.md index aa41159..828cfd4 100644 --- a/docs/model_tuning.md +++ b/docs/model_tuning.md @@ -29,28 +29,30 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. chronologically for 100% KV prefix cache stability and agentic reasoning continuity). To strip past `` blocks instead, set `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower - KV cache hit rate. v22 also honours `preserve_reasoning`, so `--reasoning-preserve` + KV cache hit rate. It also honours `preserve_reasoning`, so `--reasoning-preserve` (`common/arg.cpp:3677-3689`) works as the CLI equivalent. Path is repo-relative, so `llama-server` must be launched from the repo root — `read_file()` resolves against the process CWD, not the INI file's directory. `Qwen3-Coder-Next` entries deliberately keep their GGUF-embedded template; froggeric's README claims compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. -- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entry pins - `xhigh`.** v22 defaulted the level to `xhigh` for *every* model the template serves, injecting - a ~45-token "Reasoning effort is set to xhigh..." paragraph at the top of the system prompt. - v22.1 moved that default to `medium`, the one level that injects no instruction text at all. - Qwen 3.6 has no trained notion of the concept, so its entries pin `medium` — under v22 that - overrode a wrong default, under v22.1 it restates the right one, and either way the pin is what - makes the level independent of the template version. Qwen 3.8 *is* trained on it and Qwen's own - template defaults to `xhigh`, so that entry pins `xhigh` explicitly: leaving the key unset - silently downgraded it to `medium` at the v22.1 bump. Never rely on the template default here. - `--reasoning-effort` writes only a template kwarg (`common/arg.cpp:3650-3660`), which a request - can still override (`tools/server/server-common.cpp:1312-1319`), as can a `<|think_low|>` / - `<|think_medium|>` / `<|think_xhigh|>` tag typed inside a message (new in v22.1, stripped before - rendering). Unlike the GGUF-embedded 3.8 template, the vendored one never raises on an unknown - level — v22.1 maps `high` and `max` to `xhigh`, `minimal` to `low`, `none` to thinking off, and - anything else *down* to `medium` rather than up to `xhigh` as v22 did. +- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entries pin + `xhigh`. Never leave the key unset.** The level is one injected paragraph at the top of the + system prompt — ~45 tokens of "Reasoning effort is set to xhigh..." — and `medium` is the single + level that injects nothing at all (`chat_template.jinja:53-59`). Qwen 3.6 has no trained notion of + the concept, so its entries pin `medium`; Qwen 3.8 *is* trained on it and Qwen's own template + defaults to `xhigh`, so those pin `xhigh`. The template's own default is a + `_default_reasoning_effort` variable at the top of the file (`chat_template.jinja:17`), currently + `medium`, and it has moved between template releases before. That is the whole reason to pin: + an unpinned entry silently changes reasoning level at a template bump, and a pinned one renders + byte-identically across bumps. `--reasoning-effort` writes only a template kwarg + (`common/arg.cpp:3650-3660`), which a request can still override + (`tools/server/server-common.cpp:1312-1319`), as can a `<|think_low|>` / `<|think_medium|>` / + `<|think_xhigh|>` tag typed inside a message (stripped before rendering). Unlike the + GGUF-embedded 3.8 template, the vendored one never raises on an unknown level: it maps `high`, + `max`, `ultracode` and `extreme` to `xhigh`, `minimal` to `low`, `none` and `off` to thinking + off, and anything else *down* to `medium` (`chat_template.jinja:21-26`). The `ultracode` and + `extreme` aliases exist for Claude Code, Cursor and Cline. - **`xhigh` is kept on `Qwen3.8-27B`, and `--reasoning-budget` — not a lower level — is the guard rail for it.** Qwen publishes no per-level benchmarks; every number on the 27B card is at the @@ -71,13 +73,13 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. the block. Set it if an agentic client that sends its own `max_tokens` starts returning empty content; a request can override it per call via `reasoning_budget_tokens`. -- **`Qwen3.8-27B` pins the template too — v22 removed the reason it used to be the exception.** +- **`Qwen3.8-27B` pins the template too, even though its embedded one is already the newer file.** Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, `eos = 248046`), but it embeds a *different, newer* 8952-byte template, not the 7764-byte one shared by Qwen3.6-27B and both Bonsai variants. That newer file already - defaults `preserve_thinking` on and adds `reasoning_effort`, which is why the entry - shipped unpinned until v22 supported 3.8. Three defects justify pinning anyway, all + defaults `preserve_thinking` on and adds `reasoning_effort`, so it is a plausible + candidate for going unpinned. Three defects rule that out, all reproducible by rendering the embedded template directly: tool calls whose `arguments` arrive as a JSON *string* (what most OpenAI-compatible clients send) abort with `Can only get item pairs from a mapping`; history carrying reasoning inside `content` @@ -85,25 +87,41 @@ relevant section on demand. Cross-model rules are in `docs/presets.md`. of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. - v22.1 handles all three and its `xhigh` instruction text is byte-identical to the - official one, so `reasoning = on` with `reasoning-effort = xhigh` reproduces the - pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is + The vendored template handles all three and its `xhigh` instruction text is + byte-identical to the official one, so `reasoning = on` with `reasoning-effort = xhigh` + reproduces the pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is selected purely on `` + `` block. The pin additionally brings froggeric's agentic extras (two-tier tool-error escalation, `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). -- **v22 fixed two v19 deviations from the official Qwen prompt format, so the bump changed - every pinned entry, not just the new 3.8 one.** v19 serialized `` entries - *unwrapped* (`{"description": ..., "name": ..., "parameters": ...}`); v22 emits the - wrapped OpenAI form (`{"function": {...}, "type": "function"}`), which is what - Qwen3.6-27B's own 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 - was the outlier. v19 also rendered `\n` before assistant content where the - official templates and llama.cpp's own generation prompt use `\n\n` - (`common/chat.cpp:1163`). Both are corrections, but they change prompt bytes, so the - bump invalidates existing KV prefix caches once; `reasoning-effort = medium` suppresses - only the new steering paragraph and does not restore v19 output. +- **The vendored template (v22.4, pinned at `e649070`) corrects two v19 deviations from the + official Qwen prompt format, so moving to it changed every pinned entry, not just the new 3.8 + ones.** v19 serialized `` entries *unwrapped* + (`{"description": ..., "name": ..., "parameters": ...}`); the current template emits the wrapped + OpenAI form (`{"function": {...}, "type": "function"}`), which is what Qwen3.6-27B's own + 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 was the outlier. v19 also + rendered `\n` before assistant content where the official templates and llama.cpp's own + generation prompt use `\n\n` (`common/chat.cpp:1163`). Both are corrections, but they + change prompt bytes, so adopting the template invalidates existing KV prefix caches once; + `reasoning-effort = medium` suppresses only the steering paragraph and does not restore v19 + output. + +- **The template emits an empty `\n\n` before a historical tool call whose assistant + message carried no reasoning, and that is deliberate.** Qwen itself emits a think block before a + tool call when thinking is on, so injecting an empty one keeps rendered history token-aligned + with the model's own generation. It fires only when the client drops reasoning on the round trip; + supplying `reasoning_content`, `thinking`, `message.reasoning` (the vLLM and Responses API + spelling) or an inline `` block suppresses it. This is *not* the "empty think poisoning" + the template's README calls out — that was replacing *real* thoughts with empty blocks to save + tokens, which this template does not do. Verified by rendering the template against plain chat, + system-prompt, thinking-off, multi-turn-with-thinking, vision, and both tool-argument wire + formats; the tool-call case is the only one where an unset `reasoning_content` changes the + output. The same release line also brings the effort aliases above, reasoning de-duplication when + a client populates both `reasoning_content` and an in-content ``, complete serialization of + scalar and list tool arguments, and single-newline separation between consecutive `` + blocks for token parity on multi-tool turns. - **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself @@ -238,16 +256,39 @@ block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 exp 112.219 MiB rather than the 14.5 MiB a DSV4 checkpoint costs, and because `slot.prompt.checkpoints` is per slot (`tools/server/server-context.cpp:2283`) the host budget is `parallel x ctx-checkpoints x 112 MiB`. At `parallel = 4`, 32 checkpoints would reserve up to - 14.3 GiB of host RAM; 8 holds it at the ~3.6 GiB that `parallel = 1` with 32 would have cost. - Raising `parallel` again means lowering this in step. - -- **Measured on a 24463 MiB card at `ctx-size = 262144` / `parallel = 1` / `q8_0` K + V, CLIP on - CPU: 20174 MiB used, 3964 MiB free, 19.87 t/s tg at short context.** The free figure tracks - `fit-target = 3072` plus ~0.9 GiB of slack, so the margin is doing its job and is not obviously - over-provisioned. Throughput is expert-traffic bound, not attention bound: every token reads - 10 of 512 experts across all 48 layers, ~26.1 MiB per layer at IQ4_XS, so the ~1.25 GiB per token - that is not resident on the GPU is what sets the rate. That is the currency `ctx-size` is spent - in — 1,300 MiB of KV is one expert layer is roughly 2 % of tg. + 14.3 GiB of host RAM; 8 holds it at the ~3.6 GiB that `parallel = 1` with 32 would have cost, and + at the shipped `parallel = 2` it is ~1.8 GiB. Raising `parallel` again means lowering this in step. + +- **Measured on a 24463 MiB card, `q8_0` K + V, CLIP on CPU.** At `ctx-size = 262144` / + `parallel = 1`: 20174 MiB used, 3964 MiB free, 19.87 t/s tg at short context. At the shipped + `ctx-size = 524288` / `parallel = 2`: 19950 MiB used, 4513 MiB free, 14.90 t/s on a 400-token + prose completion. The two throughput figures are *not* a controlled comparison — different + prompts, and `ngram-mod` acceptance dominates on predictable output (the same config returns + 22.51 t/s counting to 60). "Used" barely moves between configs because `fit` always fills to the + `fit-target` margin; what changes is the composition. Throughput is expert-traffic bound, not + attention bound: every token reads 10 of 512 experts across all 48 layers, ~26.1 MiB per layer at + IQ4_XS, so the ~1.25 GiB per token that is not resident on the GPU is what sets the rate. That is + the currency `ctx-size` is spent in — 1,300 MiB of KV is one expert layer is roughly 2 % of tg. + +- **`ctx-size = 524288` with `parallel = 2` buys two concurrent full-length conversations, and + costs about half the GPU-resident expert layers to do it.** `llama-fit-params` gives the fixed + cost directly, in MiB, as a function of `n_ctx_seq` (`context` is KV plus the recurrent rows, + `compute` is the graph buffer): + + | `n_ctx_seq` | context | compute | fixed total | + |------------:|--------:|--------:|------------:| + | 65,536 | 1,234 | 573 | 1,807 | + | 262,144 | 4,600 | 1,821 | 6,421 | + | 524,288 | 9,088 | 3,497 | 12,585 | + | 1,048,576 | 18,064 | 6,829 | 24,893 | + + Add 112 MiB per sequence beyond the first for the extra recurrent row. So the shipped config's + fixed cost is 12,697 MiB against 6,758 MiB for 262144 / `parallel = 4`; the ~5,900 MiB difference + comes straight out of expert layers, taking them from ~8.3 of 48 to ~4.4. The benefit is real but + narrow: under `kv-unified` `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`) while the server + still caps each *slot* at `n_ctx_train = 262144`, so a single conversation is capped at 262144 + either way and only a *second* concurrent long conversation can reach into the extra cells. On a + single-user workload the larger pool is paid for and unused. - **A 1,048,576-cell pool does not fit, and the ceiling is arithmetic rather than a tuning question.** Going from 262144 to 1048576 takes the KV cache from 4,488 to 17,952 MiB at `q8_0`, @@ -256,7 +297,8 @@ block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 exp (`src/llama-memory-hybrid-idx.h:132-138`), so the compute buffer grows by the same factor of four on top. Even if it squeezed in it would be a regression, because zero expert layers on the GPU is strictly slower than the current split. `parallel` is not a way around it: under `kv-unified` the - slots share one pool, so "four slots of 262144" *is* 1,048,576 cells at identical cost. The only + slots share one pool, so wanting four slots that can each reach 262144 *is* asking for 1,048,576 + cells, at exactly the cost tabulated above. The only route to a 1 M pool on 24 GB is `q4_0` K and V, which halves it to 9,504 MiB, and `cache-type-k` is precisely the value that should not drop because it types the indexer. Note that a large pool needs no `override-kv`: `n_ctx_train` is 262144, so the server caps each slot there @@ -272,7 +314,7 @@ block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 exp force-disables context shift *and* cache-reuse with two warnings at `tools/server/server-context.cpp:1185-1195` — both are expected on startup, not a misconfiguration. Speculative rollback then goes through checkpoints - (`tools/server/server-context.cpp:1224-1226`), so `ctx-checkpoints = 32` is required for + (`tools/server/server-context.cpp:1224-1226`), so a non-zero `ctx-checkpoints` is required for `ngram-mod` to be useful rather than being an optimisation. `swa-full` is inert: `swa_type` is `NONE`, which is why the model takes the `hybrid_idx` path at `src/llama-model.cpp:2502` at all, and the server clears the flag at `:1197-1202`. diff --git a/docs/presets.md b/docs/presets.md index faa0e3e..5229ce1 100644 --- a/docs/presets.md +++ b/docs/presets.md @@ -23,6 +23,22 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. +## fit-target + +- **The `fit-target` values in these files assume the dGPU does not drive the display, and must be + raised on a machine where it does.** `fit` takes a single `cudaMemGetInfo` snapshot at t=0 + (`common/fit.cpp:194`) and carries no WDDM or framebuffer allowance of its own, so the margin is + the only thing standing between the fitted split and the untracked consumers — the CUDA VMM + scratch pool, the lazy cuBLAS workspace, CUDA graph instances, and anything the compositor takes. + On the development machine the desktop runs on the Intel iGPU, so the NVIDIA card reports + 23151 of 24462 MiB free with only ~1.3 GiB reserved and `fit-target = 3072` is covering CUDA and + WDDM scratch alone. Drive a display from the same card and a browser, a compositor and a second + monitor come out of that same margin. Verify with `llama-server --list-devices`, which prints the + free figure fit will actually work from, and confirm the split is on the intended adapter with + `Get-Counter "\GPU Process Memory(*)\Dedicated Usage"` — the instance name carries both the pid + and the adapter LUID, so it distinguishes a dGPU allocation from an iGPU one that `nvidia-smi` + cannot see at all. + ## load-mode - **All entries use `load-mode = dio`; never pair `direct-io` with `no-mmap` again.** Both spellings diff --git a/presets/README.md b/presets/README.md index 49de12b..ba8c9e6 100644 --- a/presets/README.md +++ b/presets/README.md @@ -82,8 +82,9 @@ The section header (e.g. `[gemma-4-31B-it.IQ4_XS.gguf]`) is the model name clien > entries `xhigh`. `medium` is the one level that injects no instruction text into the system > prompt, and Qwen 3.6 has no trained notion of the concept; Qwen 3.8 *is* trained on it and > `xhigh` is what Qwen's own template defaults to. Both are pinned rather than left unset because -> the vendored template's default has moved before (v22 shipped `xhigh`, v22.1 `medium`). Clients -> can still override per request via the OpenAI `reasoning_effort` field. +> the vendored template's own default has moved between its releases, so an unpinned entry would +> silently change reasoning level at a template bump. Clients can still override per request via +> the OpenAI `reasoning_effort` field. > > All `gemma-4-*` entries set `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja` — > the official Google template bundled with llama.cpp itself, kept in lock-step with diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 98e91e6..51573c4 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -174,8 +174,8 @@ model = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\Qwen3.8-Flash-Next.IQ4_XS.gguf mmproj = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\mmproj.Qwen3.8-Flash-Next.Q8_0.gguf no-mmproj-offload = true image-min-tokens = 1024 -ctx-size = 262144 -parallel = 4 +ctx-size = 524288 +parallel = 2 n-gpu-layers = -1 cache-type-k = q8_0 cache-type-v = q8_0 diff --git a/vendor/Qwen-Fixed-Chat-Templates b/vendor/Qwen-Fixed-Chat-Templates index 9f14778..e649070 160000 --- a/vendor/Qwen-Fixed-Chat-Templates +++ b/vendor/Qwen-Fixed-Chat-Templates @@ -1 +1 @@ -Subproject commit 9f14778c92c3b5ed3e0738085694c0d3452802dd +Subproject commit e6490706c74d558c2020b9360e556451488a348b From b237b008a1750059d97208c8fa19329fd6cf003f Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Fri, 28 Aug 2026 17:18:30 +0200 Subject: [PATCH 19/20] Drop the fit-target section from docs/presets.md and fix two errors in its load-mode section The fit-target section restated docs/model_tuning.md:445-459 - the cudaMemGetInfo snapshot at t=0, the absent WDDM and framebuffer allowance, and the untracked CUDA VMM scratch, cuBLAS workspace and graph instances - in a second file, with the same citations and no link between them, so the two would have drifted apart at the first change. The one fact it added, that the desktop runs on the Intel iGPU and the card therefore reports 23151 of 24462 MiB free, does not change what a reader does. model_tuning.md already records the measured consequence: 1368 MiB left for untracked consumers at the default margin against 3497 at fit-target 3072, and one extra expert layer on the CPU as the price. It was written to make a commit message true rather than because the documentation needed it. Removing it surfaced two defects in the adjacent load-mode section, both introduced when the Qwen3.8-Flash-Next entry landed. The section still opened with "All entries use load-mode = dio" immediately above the bullet explaining that Qwen3.8-Flash-Next must not, and it put the per-token PLE gather at 8 rows where it is 16: ple_n_heads is (ngram_size - 1) * heads_per_ngram, 2 * 8, at src/models/qwen4exp.cpp:64. model_tuning.md already carried the corrected figure, so the two files disagreed. --- CHANGELOG.md | 1 - docs/presets.md | 21 +++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0683a53..8140c94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,7 +46,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Document the measured Qwen3.8-Flash-Next 24 GB VRAM and throughput figures - [Documentation] Document the measured Qwen3.8-Flash-Next context and compute cost per n_ctx_seq - [Documentation] Document why a 1M cell pool does not fit Qwen3.8-Flash-Next on 24 GB -- [Documentation] Document that the fit-target values assume the dGPU does not drive the display ### Changed - [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md diff --git a/docs/presets.md b/docs/presets.md index 5229ce1..66e932d 100644 --- a/docs/presets.md +++ b/docs/presets.md @@ -23,25 +23,10 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. -## fit-target - -- **The `fit-target` values in these files assume the dGPU does not drive the display, and must be - raised on a machine where it does.** `fit` takes a single `cudaMemGetInfo` snapshot at t=0 - (`common/fit.cpp:194`) and carries no WDDM or framebuffer allowance of its own, so the margin is - the only thing standing between the fitted split and the untracked consumers — the CUDA VMM - scratch pool, the lazy cuBLAS workspace, CUDA graph instances, and anything the compositor takes. - On the development machine the desktop runs on the Intel iGPU, so the NVIDIA card reports - 23151 of 24462 MiB free with only ~1.3 GiB reserved and `fit-target = 3072` is covering CUDA and - WDDM scratch alone. Drive a display from the same card and a browser, a compositor and a second - monitor come out of that same margin. Verify with `llama-server --list-devices`, which prints the - free figure fit will actually work from, and confirm the split is on the intended adapter with - `Get-Counter "\GPU Process Memory(*)\Dedicated Usage"` — the instance name carries both the pid - and the adapter LUID, so it distinguishes a dGPU allocation from an iGPU one that `nvidia-smi` - cannot see at all. - ## load-mode -- **All entries use `load-mode = dio`; never pair `direct-io` with `no-mmap` again.** Both spellings +- **Every entry but `Qwen3.8-Flash-Next` uses `load-mode = dio`; never pair `direct-io` with + `no-mmap` again.** Both spellings are deprecated, and they write the *same* mutually exclusive enum — `--no-mmap` sets `LLAMA_LOAD_MODE_NONE` (`common/arg.cpp:2594`) while `--direct-io` sets `LLAMA_LOAD_MODE_DIRECT_IO` (`:2603`) — so setting both means only whichever is parsed last wins. @@ -56,7 +41,7 @@ VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini `per_layer_token_embd` n-gram hash table is created with `TENSOR_READ_LAZY` (`src/models/qwen4exp.cpp:139-140`) and the loader gates that flag on `use_mmap` (`src/llama-model-loader.cpp:1290`), which every non-mmap `load-mode` clears (`:559`). A session - touches 8 of the table's 320 million rows per token, so under `mmap` the resident working set + touches 16 of the table's 320 million rows per token (`ple_n_heads`, `:64`), so under `mmap` the resident working set stays in the hundreds of MiB while `dio` reads and holds all 26.8 GiB. `no-host = true` is part of the same mechanism, not an independent choice — see `docs/model_tuning.md` -> *Qwen3.8-Flash-Next*. From 426fc24ec3ad059a18f6467c0f4a1621f485ee79 Mon Sep 17 00:00:00 2001 From: Finn Kumkar Date: Fri, 28 Aug 2026 18:35:20 +0200 Subject: [PATCH 20/20] Size the gguf-py row group by bytes instead of a fixed 16 rows _apply_over_grouped_rows split rows into groups of 16 rows, so the group's byte size followed the row width. The replacement grows the group until it holds 160 KiB of input but never goes below 16 rows, leaving np.array_split and np.concatenate(..., out=out) untouched. Full scale on Windows with numpy 2.2.6, a 640-wide f32 row goes 129.4 s -> 4.8 s (27.0x); across an 18-point width ladder from 128 to 16384 the patched form is flat at ~1.13-1.18 s where upstream ranges 1.12-5.13 s. The cause is the Windows low-fragmentation heap and the trigger is the retained result, not the working buffer. Microsoft documents that the LFH does not serve allocations above about 16 KiB; once the retained group result crosses that, it interleaves with the next group's transient temporaries, fragments the general heap, and per-allocation cost grows roughly with the square of the live block count. Measured 3.7 us/alloc flat from 1 K to 65 K live blocks at 8 KiB retained, against 13 -> 526 us/alloc over the same range at 32 KiB. The cliff was located by varying the result-to-temporary size ratio and watching it move with the retained size, staying pinned to the documented ceiling. A pure allocator microbenchmark with no gguf code reproduces it, and the same script on glibc is flat with no band at any size. The 16-row floor is deliberate. Without it the budget asks for fewer than 16 rows once a f32 row exceeds 2560 elements, which regresses Linux by about 1.2x at widths 4096-8192, reproduced on WSL1 and on a Debian 13 / kernel 6.12 / glibc 2.41 VM at repeats 7 and 15. The floor makes the change a no-op for every width the budget cannot improve, so it cannot be slower than upstream anywhere. The price is Windows throughput at wide rows: 7.57 s against 4.79 s unfloored at 5120-wide full scale. Three alternatives were measured and rejected. A larger fixed row count only relocates the fault, since the pathology follows rows x width x itemsize: 64 rows fixes 640-wide and breaks 160-wide. Writing each group straight into out regresses 2.2x on numpy 1.26.4, which upstream CI pins, and 2.3x on glibc for wide rows. Bounding the retained block to 16 KiB directly, which the mechanism suggests, loses 1.18-1.24x to the input budget at every scale. The previous patch also dropped a np.uint64 widening in BF16.quantize_blocks. That hunk is worth 1.41x on Windows and 1.84x on glibc but is a separate concern touching a line upstream added on purpose in #7843, so it is no longer carried here. Verified byte-for-byte against the C implementation through gguf-py/tests/test_quants.py (25 types, 60 exact matches, 0 mismatches), across 10 type and width combinations in both directions on Windows/numpy 2.2.6, Windows/numpy 1.26.4 and glibc/numpy 2.2.6, and over 15 adversarial shapes covering a single row, row counts below one group, prime row counts that leave a remainder, and rows up to 262144 wide. Assisted-by: Claude Opus 4.6 --- AGENTS.md | 2 +- CHANGELOG.md | 5 +-- docs/build_system.md | 12 ++++-- ...001-gguf-py-size-row-groups-by-bytes.patch | 19 ++++++++++ ...01-gguf-py-write-row-groups-directly.patch | 37 ------------------- 5 files changed, 31 insertions(+), 44 deletions(-) create mode 100644 patches/0001-gguf-py-size-row-groups-by-bytes.patch delete mode 100644 patches/0001-gguf-py-write-row-groups-directly.patch diff --git a/AGENTS.md b/AGENTS.md index 8adc264..48f9049 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,7 @@ Other helpers in `examples/`: `count_tokens.ps1`, `benchmark.ps1` (perplexity), - **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an idempotent OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt`; `.gitmodules` sets `ignore = dirty` for it. Don't "clean it up." `docs/build_system.md` -> *Submodule lifecycle* - **Each build wipes `vendor/llama.cpp`** back to `origin/master` then checks out the requested `-version` / PR, so local edits there are lost by design. Other submodules are never advanced by the build script and must be bumped by hand. `docs/build_system.md` -> *Submodule lifecycle* -- **`./patches/*.patch` is the only supported way to carry a local change into the submodule.** It is re-applied after every checkout and the build aborts if a patch no longer applies. Currently one patch, which keeps GGUF conversion of large short-row tensors from running 26x too slow. `docs/build_system.md` -> *Local patches* +- **`./patches/*.patch` is the only supported way to carry a local change into the submodule.** It is re-applied after every checkout and the build aborts if a patch no longer applies. Currently one patch, which keeps GGUF conversion from stalling when a tensor's row width puts the quantizer group on a slow allocation size. `docs/build_system.md` -> *Local patches* - **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. - **Three vendored paths are hardcoded** (`gguf_dump.py`, `speed-bench/`, `models/templates/`). Upstream has moved them before; after a version bump treat a startup failure naming one as a relocation first. `docs/build_system.md` -> *Upstream path dependencies* - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. diff --git a/CHANGELOG.md b/CHANGELOG.md index 8140c94..e09e42d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Build] Apply patches from ./patches to the llama.cpp submodule after checkout - [Build] Abort the build when a local patch no longer applies - [Documentation] Document the local patch mechanism in the build system reference -- [Documentation] Document the gguf-py row-group fix, how it was verified, and that it is a throughput fix rather than the PLE table OOM fix +- [Documentation] Document the gguf-py row-group fix, the platforms it was measured on, and the three variants that were rejected - [Documentation] Document the Qwen3.8-Flash-Next per_layer_token_embd table and its lazy read path - [Documentation] Document why the Qwen3.8-Flash-Next entry needs load-mode = mmap instead of dio - [Documentation] Document why no-host keeps the Qwen3.8-Flash-Next mmap aliasing path reachable @@ -69,8 +69,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Documentation] Correct the GGML_CUDA_FA_ALL_QUANTS failure mode from an abort to a silent CPU fallback - [Documentation] Correct the claim that Muse-Glimmer-30B has no bundled chat template - [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections -- [Conversion] Fix BF16 GGUF conversion of large short-row tensors running 26x too slow -- [Conversion] Drop the uint64 widening in the gguf-py BF16 quantizer +- [Conversion] Fix GGUF conversion stalling when a tensor row width puts the quantizer group on a slow allocation size ## [1.40.0] - 2026-08-05 diff --git a/docs/build_system.md b/docs/build_system.md index a1c5292..ba01140 100644 --- a/docs/build_system.md +++ b/docs/build_system.md @@ -14,11 +14,17 @@ sections back are in AGENTS.md "Non-obvious behavior". - **`./patches/*.patch` are re-applied to `vendor/llama.cpp` on every build.** The step runs after the `-version` / PR checkout (`rebuild_llama.cpp.ps1:254-284`), because the checkout would otherwise discard them. It needs no idempotence guard like the OpenBLAS shim — the `reset --hard` earlier in the script guarantees a clean tree. Applied with `git apply --3way`, so a patch still lands when upstream moves the lines *around* a hunk; if it fails, upstream moved the patched code itself and the script `throw`s rather than silently building an unpatched tree. `--3way` implies `--index`, which would leave the files staged and make `git checkout -- ` restore the *patched* copy instead of the upstream one — the trailing `git reset --quiet` unstages them so they show up as plain worktree modifications, same as the shim. -- **`0001-gguf-py-write-row-groups-directly.patch` fixes a 26x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) collected every 16-row group in a Python list and merged them with a single `np.concatenate(..., out=out)`. That holds every group alive until the merge, so peak intermediate memory equals the whole output, and because the group is fixed at 16 *rows* regardless of row length, cost per byte tracks the group count rather than the byte count. The copy count is **not** what changes: both forms allocate one array per group and copy it into `out` once, so the win is peak memory and cache locality, not a saved copy. Measured on Qwen3.8-Flash-Next `blk.N.ffn_down_exps.weight` (512 x 2560 x 640 f32, 1.68 GB out), BF16 conversion ran at **13.2 MB/s** — against 348.8 MB/s for the identical byte count in the `ffn_gate/up_exps` layout (512 x 640 x 2560), which has 4x fewer groups. Writing each group straight into `out`, and sizing groups by bytes (160 KiB of *input*, so one group stays in L2 at any row length; the dequantize direction produces a proportionally larger group), gives a uniform ~500 MB/s: 497.9 / 505.9 / 490.8 MB/s on the down, gate-up and `per_layer_token_embd` shapes, i.e. 37.6x / 1.5x / 2.2x. Per expert tensor the whole stack-plus-quantize pipeline goes 126.7 s -> 4.7 s (26.7x); the residual 1.4 s is the `torch.stack` upcast, which this patch does not touch. The second hunk drops a `np.uint64` widening in `BF16.quantize_blocks` (worth a further ~21%): the NaN fixup directly above it rewrites NaN lanes to `(u.i & 0xffff0000) | 0x00400000`, bounding `n` at `0xFFFF0000` there and at `0xFF800000` (-inf) everywhere else, so `n + 0x7fff + 1 <= 0xFFFF8000` and the add cannot overflow uint32. The C reference (`ggml/src/ggml-impl.h:611-623`) gets the same guarantee from an early `return` that the vectorised form cannot use. +- **`0001-gguf-py-size-row-groups-by-bytes.patch` fixes a 27x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) split rows into groups of 16 *rows*, so the group's byte size followed the row width. The patch grows the group until it holds 160 KiB of input but never goes below 16 rows, leaving `np.array_split` and `np.concatenate(..., out=out)` untouched. Two lines of logic. Measured full scale (3.355 GB f32 in) on Windows with numpy 2.2.6: a 640-wide row goes **129.4 s -> 4.8 s (27.0x)**; 2560-wide is unchanged (the budget yields exactly 16 rows there, so the patch is a no-op by construction). Across an 18-point width ladder from 128 to 16384 the patched form is flat at ~1.13-1.18 s where upstream ranges 1.12-5.13 s. -- **It is a throughput fix, not an OOM fix, and the `per_layer_token_embd` figure is per shard.** `conversion/qwen4exp.py:145-172` wraps the 128 PLE shards in `gguf.LazyChunkedTensor`, whose `tofile` quantizes one shard at a time (`gguf-py/gguf/lazy.py:275-289`), so `_apply_over_grouped_rows` only ever sees ~1.6 GB of that table, never all 95 GiB. Peak RSS for it was solved separately by `53c0f624a` (stream the shards, ~300 GB -> one shard) and `a510c82e1` (`LazyChunkedTensor`) — don't re-credit that to this patch. `a510c82e1` also landed *after* the 2.2x measurement above, so re-measure that shape on the current submodule HEAD before quoting it anywhere, and don't pitch the patch upstream against #15623 / #15648. +- **The cause is the Windows low-fragmentation heap, and the trigger is the *retained* block, not the working buffer.** Each `func(group)` call allocates several transient temporaries and returns one result the caller retains until the whole tensor is done. Microsoft documents that the LFH does not serve allocations above ~16 KiB. Once the retained result crosses that, the interleaving of long-lived and short-lived blocks fragments the general heap and per-allocation cost grows roughly with the square of the live block count: measured 3.7 us/alloc flat from 1 K to 65 K live blocks at 8 KiB retained, against 13 -> 526 us/alloc over the same range at 32 KiB retained. The cliff was located by varying the result-to-temporary size ratio and watching it move: it sits at 17 KiB retained for a 0.5 ratio and 16.2 KiB for 0.25, pinned to the documented ceiling in both cases. Upstream's 16 rows of 640 f32 lands at 20,480 B retained, just past it. A pure allocator microbenchmark with no gguf code reproduces the whole effect (`.tmp/.../alloc_band.py`), and the same script on glibc is flat at 0.88-1.16x with no band at any size. -- **The patch is byte-for-byte equivalent to upstream, and that was verified three ways.** Against the C implementation with `python gguf-py/tests/test_quants.py --libggml build/bin/Release/ggml-base.dll` (25 types, 60 exact matches, identical output before and after apart from a shifted warning line number); across all 24 registered quant types in both quantize and dequantize directions over six shapes chosen to exercise the remainder path; and for the `uint32` kernel exhaustively over all 2^32 float32 bit patterns (0 mismatches, NaN-fixup path hit 16.7M times). Note `test_quants.py` needs `ggml-base.dll`, not `ggml.dll` — only the former exports `ggml_quantize_chunk` — and `ctypes` needs both `%CUDA_PATH%\bin` and `%CUDA_PATH%\bin\x64` added via `os.add_dll_directory` before the load will resolve. +- **The 16-row floor is deliberate and costs Windows real throughput; do not remove it.** Without it the budget asks for fewer than 16 rows once a f32 row exceeds 2560 elements, and that regresses Linux by about 1.2x at widths 4096-8192 — reproduced on WSL1 and on a real Debian 13 / kernel 6.12 / glibc 2.41 VM, twice, at repeats 7 and 15. The floor makes the patch a no-op for every width the budget cannot improve, so it can never be slower than upstream anywhere. The price is paid on Windows: at 5120-wide full scale the floored form is 7.57 s against 4.79 s unfloored (1.58x). That trade was chosen because not regressing other platforms is the harder constraint. Flipping `max(16, ...)` back to `max(1, ...)` buys the Windows win back and is a one-character change. + +- **Three alternatives were measured and rejected.** A larger fixed row count only relocates the fault, because the pathology follows `rows x width x itemsize`: 64 rows fixes 640-wide and breaks 160-wide (4.1 s -> 26.4 s). Writing each group straight into `out` instead of concatenating is faster on Windows with numpy 2.x but regresses 2.2x on numpy 1.26.4, which upstream CI pins, and 2.3x on glibc for wide rows. Bounding the *retained* block to 16 KiB directly, which the mechanism suggests, loses 1.18-1.24x to the 160 KiB input budget at every scale from 1/8 to full. A `np.uint64` widening in `BF16.quantize_blocks` is worth a further 1.41x on Windows / 1.84x on glibc and was measured but **deliberately left out** to keep this patch to one concern; it is a separate change to a line upstream added on purpose in #7843. + +- **It is a throughput fix, not an OOM fix, and the `per_layer_token_embd` figure is per shard.** `conversion/qwen4exp.py:145-172` wraps the 128 PLE shards in `gguf.LazyChunkedTensor`, whose `tofile` quantizes one shard at a time (`gguf-py/gguf/lazy.py:275-289`), so `_apply_over_grouped_rows` only ever sees ~1.6 GB of that table, never all 95 GiB. Peak RSS for it was solved separately by `53c0f624a` (stream the shards, ~300 GB -> one shard) and `a510c82e1` (`LazyChunkedTensor`) — don't re-credit that to this patch. Don't pitch the patch upstream against #15623 / #15648 either: both target peak RSS, and this patch does not change it. + +- **The patch is byte-for-byte equivalent to upstream, and that was verified three ways.** Against the C implementation with `python gguf-py/tests/test_quants.py --libggml build/bin/Release/ggml-base.dll` (25 types, 60 exact matches, 0 mismatches, exit 0); across 10 type and width combinations in both quantize and dequantize directions on Windows/numpy 2.2.6, Windows/numpy 1.26.4 and glibc/numpy 2.2.6; and over 15 adversarial shapes covering a single row, row counts below one group, prime row counts that leave a remainder, and rows up to 262144 wide (180 comparisons per numpy version, 0 failures). Note `test_quants.py` needs `ggml-base.dll`, not `ggml.dll` — only the former exports `ggml_quantize_chunk` — and `ctypes` needs both `%CUDA_PATH%\bin` and `%CUDA_PATH%\bin\x64` added via `os.add_dll_directory` before the load will resolve. ## Toolchain detection diff --git a/patches/0001-gguf-py-size-row-groups-by-bytes.patch b/patches/0001-gguf-py-size-row-groups-by-bytes.patch new file mode 100644 index 0000000..55acaa6 --- /dev/null +++ b/patches/0001-gguf-py-size-row-groups-by-bytes.patch @@ -0,0 +1,19 @@ +diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py +index 80966b6ef..85c5b49e4 100644 +--- a/gguf-py/gguf/quants.py ++++ b/gguf-py/gguf/quants.py +@@ -33,8 +33,12 @@ def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.n + for dim in oshape: + osize *= dim + out = np.empty(shape=osize, dtype=otype) +- # compute over groups of 16 rows (arbitrary, but seems good for performance) +- n_groups = (rows.shape[0] // 16) or 1 ++ # grow the group to 160 KiB, but never below 16 rows. a fixed row count ++ # makes the group size follow the row width, which is very slow for some ++ # widths on Windows ++ row_bytes = rows.shape[-1] * arr.dtype.itemsize ++ rows_per_group = max(16, (160 * 1024) // row_bytes) if row_bytes else 16 ++ n_groups = (rows.shape[0] // rows_per_group) or 1 + np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out) + return out.reshape(oshape) + diff --git a/patches/0001-gguf-py-write-row-groups-directly.patch b/patches/0001-gguf-py-write-row-groups-directly.patch deleted file mode 100644 index 6172470..0000000 --- a/patches/0001-gguf-py-write-row-groups-directly.patch +++ /dev/null @@ -1,37 +0,0 @@ -diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py -index 80966b6ef..f4495760e 100644 ---- a/gguf-py/gguf/quants.py -+++ b/gguf-py/gguf/quants.py -@@ -33,9 +33,17 @@ def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.n - for dim in oshape: - osize *= dim - out = np.empty(shape=osize, dtype=otype) -- # compute over groups of 16 rows (arbitrary, but seems good for performance) -- n_groups = (rows.shape[0] // 16) or 1 -- np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out) -+ # group by bytes, not by row count, so one group stays in L2 for any row length -+ row_bytes = rows.shape[1] * arr.dtype.itemsize -+ n_per_group = max(1, (160 * 1024) // row_bytes) if row_bytes else 1 -+ # write each group straight into out. collecting the groups for a final -+ # np.concatenate keeps them all alive, which is very slow for big tensors -+ pos = 0 -+ for start in range(0, rows.shape[0], n_per_group): -+ group = func(rows[start:start + n_per_group]).ravel() -+ out[pos:pos + group.size] = group -+ pos += group.size -+ assert pos == osize - return out.reshape(oshape) - - -@@ -209,8 +217,9 @@ class BF16(__Quant, qtype=GGMLQuantizationType.BF16): - n = blocks.view(np.uint32) - # force nan to quiet - n = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n) -- # round to nearest even -- n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16 -+ # round to nearest even. the nan fixup above caps n, so this cannot -+ # overflow and does not need to widen the whole array to uint64 -+ n = (n + np.uint32(0x7fff) + ((n >> np.uint32(16)) & np.uint32(1))) >> np.uint32(16) - return n.astype(np.uint16).view(np.uint8) - - @classmethod