diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..f4bae0b --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|compact", + "hooks": [ + { + "type": "command", + "command": "powershell -NoProfile -Command \"'SESSION_ID=' + ([Console]::In.ReadToEnd() | ConvertFrom-Json).session_id\"" + } + ] + } + ] + } +} diff --git a/.claude/settings.local.json b/.claude/settings.local.json deleted file mode 100644 index e4dcacb..0000000 --- a/.claude/settings.local.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(dir /b)", - "Bash(dir)", - "Bash(cat:*)" - ] - } -} diff --git a/.claude/skills/pr-code-review/SKILL.md b/.claude/skills/pr-code-review/SKILL.md index fa209b3..10d82ad 100644 --- a/.claude/skills/pr-code-review/SKILL.md +++ b/.claude/skills/pr-code-review/SKILL.md @@ -136,7 +136,7 @@ Do NOT flag any of the following: - Pre-existing issues not introduced in this PR's changes. - Code that appears to be a bug but is actually correct. - Pedantic nitpicks that a senior engineer would not flag. -- General code quality concerns unless explicitly required in AGENTS.md. +- General code quality concerns unless explicitly required in AGENTS.md or `docs/`. - Issues explicitly silenced in code (e.g., via a lint ignore comment). - Pure code style or formatting preferences. - Potential issues that depend on specific inputs or runtime state. @@ -148,8 +148,14 @@ Do NOT flag any of the following: These are architectural and safety concerns specific to this PowerShell wrapper around llama.cpp; there is no linter to catch them. Check for these during all passes, in addition to general defect scanning. They are NOT style issues; they -are correctness and safety rules. Each maps to a rule in AGENTS.md -("Non-obvious behavior", "Presets", "Changelog style"). +are correctness and safety rules. + +This checklist deliberately restates rules whose full rationale lives in +`docs/build_system.md`, `docs/presets.md` and `docs/model_tuning.md`, and whose +one-line form lives in AGENTS.md ("Non-obvious behavior", "Traps", "Changelog +style"). The duplication is intentional: a review pass must stay self-contained +and must not depend on on-demand doc reads mid-pass. When a rule changes, all +three locations move together. - **Ephemeral submodule edits:** Changes under `vendor/llama.cpp/` that expect to persist. Each `rebuild_llama.cpp.ps1` resets the submodule to diff --git a/.claude/skills/session-retrospective/SKILL.md b/.claude/skills/session-retrospective/SKILL.md new file mode 100644 index 0000000..9d1f304 --- /dev/null +++ b/.claude/skills/session-retrospective/SKILL.md @@ -0,0 +1,241 @@ +--- +name: session-retrospective +description: > + End-of-session retrospective. Use when the user asks to review the + current session for improvement opportunities, or types + /session-retrospective. Scans the conversation for friction and discoveries, + classifies each candidate by destination (AGENTS.md, docs/, new + skill, existing skill, drop), runs an adversarial gauntlet that + kills candidates failing already-captured / counterfactual / + generalization / triviality / process-worked tests, then presents + only survivors interactively. Default expected count is zero. + Single-session scope; no cross-session memory. +disable-model-invocation: true +--- + +# Skill: Session Retrospective + +End-of-session retrospective that proposes additions to `AGENTS.md`, +`docs/`, an existing skill, or a new skill, but only when an +adversarial gauntlet has killed every candidate that does not survive +five concrete tests. Default expected output: zero additions. + +## Core principle + +The skill argues against itself before bothering the user. The +default-on bias of an LLM proposing documentation is to over-propose: +"plausible improvements" rather than "improvements that would have +prevented this session's friction". Without an adversarial pass the +user has to do the culling manually with a single dismissive question; +the gauntlet automates that pushback. + +A finding survives only when it passes **all five** gauntlet tests. +The agent must justify each survivor *up* from zero with concrete +evidence, not *down* from a cap. + +The `process-worked` test specifically asks whether an **automated** +process caught the issue, not whether the user did. Treating a human +"wait, why are you doing X?" as `process-worked` collapses the gauntlet +into "the user can override anything, so propose nothing", which is the +failure mode the core principle warns about. The user catching the +issue is evidence the issue is real, not evidence the system already +handled it. + +## When to use + +- The user asks for a retro of the current session. +- The user types `/session-retrospective`. + +Do **not** use this skill: + +- Mid-session ("we just learned X — write it down" is a normal Edit, + not a retro). +- Across multiple sessions. Single-session scope; no friction log, no + recurrence tracking, no cross-session memory. + +## Workflow + +Track the steps with `TodoWrite`. Mark each `in_progress` when +starting and `completed` when done. + +1. **Resolve context.** + - Current branch via `git branch --show-current`. This repo has no + ticket-key convention; the branch name is the report label, with + `/` replaced by `-`. + - Timestamp: `Get-Date -Format "yyyyMMdd-HHmm"`. +2. **Scan session.** Walk the conversation in order. Tag every event + that is one of: + - **Friction:** user corrected output, asked to redo, expressed + frustration, overrode an approach, or 3+ rounds on the same + topic. Distinguish friction from normal iteration ("try a smaller + ctx-size" is collaboration, not friction). + - **Discovery:** a fact about this wrapper, the build, or a model's + behaviour that was learned mid-session and is not yet captured. + - **Procedure:** a multi-step playbook (3+ steps with decision + logic) that was executed and is likely to recur. + - **Skill gap:** a task adjacent to an existing skill where the + skill could have helped but didn't, or didn't exist. +3. **Classify by destination** using the routing table below. +4. **Self-challenge gauntlet.** Run every candidate through all five + tests below. A candidate survives only when it passes all five. + Maintain a kill log throughout: every killed candidate with the + test that killed it and, where relevant, the existing line that + already covers the fact. +5. **Prioritize survivors.** Default expected count is **zero**. Hard + cap **3**. Each survivor must come with a one-sentence statement + of the future-session mistake it prevents. If that sentence cannot + be written cleanly, the candidate goes back through the gauntlet + and almost always dies. +6. **Read existing destinations** for each survivor to detect last-mile + duplicates, contradictions, and the right insertion point. +7. **Present interactively, one at a time.** For each survivor: + - Quote the session evidence. + - Show the proposed concrete diff (exact text, exact location). + - State the future-session mistake it prevents. + - Ask confirm / reject / refine. +8. **Apply confirmed edits** via `Edit` / `Write`. Skip rejected ones. +9. **Write the report** to + `.tmp/sessions//retro-.md`. Report shape is in + the *Report* section below. +10. **Print the terminal output** of the turn. Exactly one of: + - `No additions needed.` + the kill log + the report file link. + - `N proposed addition(s).` (1–3) + the interactive presentation + + the kill log of dropped candidates + the report file link. + + The report file link is the last thing printed. Nothing follows + it. Any follow-up question opens a new turn. + +## Routing table + +| Candidate signal | Destination | +| --------------------------------------------------------------------------- | ------------------------------------------------------ | +| Behavioural rule about the build, the scripts, or the repo layout | `AGENTS.md` -> *Non-obvious behavior* | +| Prohibition whose violation is a silent OOM, silent corruption, or an abort | `AGENTS.md` -> *Traps*, **and** the backing `docs/` section | +| Build-script, toolchain, submodule, or vendored-path reasoning | `docs/build_system.md` | +| Cross-model INI rule (device pinning, `load-mode`, context sizing, spec) | `docs/presets.md` | +| Per-model rationale or a measured VRAM / throughput number | `docs/model_tuning.md` | +| User-facing launch or INI-syntax instruction | `presets/README.md` | +| 3+ step playbook with decision logic, fits an existing skill's scope | Extend `.claude/skills//SKILL.md` | +| 3+ step playbook that's its own concern | New `.claude/skills//SKILL.md` | +| Tool/MCP gap | Note in report only; no auto-edit | +| Session-specific minutiae unlikely to generalize | **Drop**; note in kill log | + +Some rules deliberately live in more than one place. A trap is stated +in `AGENTS.md` *Traps* without rationale, in its `docs/` section with +rationale, and — where a reviewer needs it without an on-demand read — +in the `pr-code-review` *Project-Specific Review Checklist*. When a +candidate targets one of those, propose the edit to **all** copies in a +single survivor, never to one in isolation. + +## Self-challenge gauntlet + +A candidate survives only when it passes **all five** tests. + +| Test | Operationalized as | Kill if | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| Already-captured | Run `rg -i "" AGENTS.md docs/ presets/README.md .claude/skills/` and `git log -p -n 20 -- AGENTS.md docs/ .claude/skills/` for in-session commits. | Any hit covers the same fact. Quote the existing line in the kill log. | +| Counterfactual | Locate the actual mistake event. Did the user correct a fact the agent **didn't know**, or one it **knew but lapsed on**? | If lapse, kill: more docs don't fix lapses. | +| Generalization | Would this be equally true of any llama.cpp build, or does it depend on this Windows / MSVC / CUDA / Conda wrapper and its pinned presets? | If it holds anywhere, kill: generic llama.cpp or PowerShell knowledge. | +| Triviality | Can any agent derive this in <60 seconds from `--help`, the INI files, or the build script? | If yes, kill: permanent context noise must clear a higher bar. | +| Process-worked | Did an **automated** process catch the issue *in this session*? See the scope note below — it is narrow in this repo. The user reading agent output and pushing back does NOT count. | If yes, kill: the existing automated process is the doc. | + +**Scope note on `process-worked`.** This repo has no tests, no linter, +no type-checker and no CI (`AGENTS.md` -> *Commands*). The only automated +gates are the build itself (`rebuild_llama.cpp.ps1`), the running +build-tree-process abort, and llama.cpp's own startup validation — a +refused `cache-type` pair, an unknown `load-mode` value, a missing +template file. This test therefore kills far fewer candidates here than +in a repo with a test suite; do not stretch it to cover a human review. +Conversely, a failure mode that llama.cpp *silently* accepts is by +definition not caught by any process, and is exactly what *Traps* exists +for. + +If the agent cannot identify which test a candidate would *fail*, the +candidate is not yet justified; kill it. + +## Friction signals + +| Signal | Example | +| ------------------------ | ---------------------------------------------------------------------- | +| User asks to redo | "No, redo this", "Try again", "That's not what I meant" | +| User corrects output | "Actually it should be X", "Change this to Y" | +| User pushback | "Is this really adding value?", "Keep it simple", "You keep doing X" | +| Excessive back-and-forth | 3+ rounds on the same topic without resolution | +| User overrides approach | "Don't do it that way", "Skip that step", "Just do X" | + +Do **not** treat as friction: + +- Normal iterative refinement. +- User exploring options. +- User changing direction (new input, not a mistake). +- A tuning loop that converges. Sweeping `ctx-size` or `fit-target` + across several launches is the documented way this repo finds a + number, not evidence of a problem. + +## Report + +Path: `.tmp/sessions//retro-.md`. + +Create `.tmp/sessions//` if it does not exist. Re-running +the retro for the same branch in the same session overwrites the prior +report; that is intentional. Branch names lowercase with `/` replaced +by `-`. + +Body: + +```markdown +# Session Retro: () + +## Survivors + + + +### : + +- **Evidence:** +- **Prevents:** +- **Diff:** +- **Decision:** confirmed | rejected | refined + +## Kill log + +| Candidate | Destination | Killed by | Note | +| -------------------- | ----------- | ----------------- | ----------------------------------------------------- | +| | | already-captured | Quote the existing line, or cite the file:line. | +| | | counterfactual | Agent knew the rule; lapse, not knowledge gap. | +| | | generalization | Generic knowledge. | +| | | triviality | Derivable in <60 s from . | +| | | process-worked | Caught in-session by . | + +## Summary + + +``` + +## Hard guarantees + +- Never writes to `AGENTS.md` / `docs/` / `presets/` / any skill + without explicit per-finding user confirmation. +- Never invents friction; uses only observable session evidence. +- Default expected survivor count is **zero**. Hard cap **3**. +- Kill log is mandatory output for every retro, even when zero + candidates survive. The kill log is part of the chat output, not + only the report file. +- Never modifies build scripts or preset INI files; docs and skills + only. A preset value that should change is a finding to report, not + an edit to make. +- Never commits or pushes. +- Single-session scope. No friction log, no recurrence tracking, no + cross-session memory. +- Report files only under `.tmp/sessions//`. Never + `.claude/`, the repo root, or `vendor/`. + +## Constraints + +- Do NOT modify source code (docs and skills only). +- Do NOT post comments, reviews, or any data to GitHub. +- Do NOT commit or push. +- Do NOT silently surface candidates that failed the gauntlet, even + if they look interesting. The user can override a kill by name + after seeing the kill log; the skill does not relitigate on its + own. diff --git a/.gitignore b/.gitignore index 1be1cc1..eb4610a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ # Local scratch directory for all agent-generated artifacts (screenshots, # diffs, trace outputs, experimental scripts). See AGENTS.md "Scratch Files". .tmp/ + +# Machine-local agent settings. The checked-in counterpart is +# .claude/settings.json. +.claude/settings.local.json diff --git a/.opencode/plugins/session-id-injector.js b/.opencode/plugins/session-id-injector.js new file mode 100644 index 0000000..c3eacb0 --- /dev/null +++ b/.opencode/plugins/session-id-injector.js @@ -0,0 +1,22 @@ +// Injects "SESSION_ID=" into the agent's system prompt every turn via +// OpenCode's experimental.chat.system.transform hook. The agent uses this +// value as in .tmp/sessions// per AGENTS.md +// "Scratch Files". The Claude Code equivalent is the SessionStart hook in +// .claude/settings.json. +// +// No package.json is needed: this file imports nothing, and OpenCode loads +// local plugins directly from .opencode/plugins/. A package.json would only +// be required to pull in external npm packages. +// +// The "experimental." prefix means OpenCode reserves the right to rename or +// remove the hook; sessionID is also optional in its signature, hence the +// guard. If either changes, the soft-fallback in AGENTS.md (mint +// YYYYMMDD-HHMMSS-) keeps things working at the cost of resume +// support. Verified against @opencode-ai/plugin 1.3.17 / opencode-ai 1.18.18. + +export const SessionIdInjector = async () => ({ + 'experimental.chat.system.transform': async (input, output) => { + if (!input.sessionID) return; + output.system.push(`SESSION_ID=${input.sessionID}`); + }, +}); diff --git a/AGENTS.md b/AGENTS.md index 9ed4255..48f9049 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,10 +1,10 @@ # AGENTS.md -This file provides guidance to coding agents (Claude Code, OpenCode) when working with code in this repository. +Canonical agent-instruction file for this repository. Both Claude Code (via the `@AGENTS.md` import in `CLAUDE.md`) and OpenCode (which reads `AGENTS.md` natively) load this file. It carries the always-on rules; deep reference documentation lives under `docs/` and is read on demand, not loaded into context (see *Reference* at the end). ## What this is -A PowerShell wrapper around upstream [llama.cpp](https://github.com/ggml-org/llama.cpp), pinned as a submodule at `vendor/llama.cpp/`. No original C/C++ lives here — only `.ps1` scripts driving CMake + MSVC + Conda. The session shell is bash on Windows; the project's own scripts must run from `pwsh`/`powershell`. +A PowerShell wrapper around upstream [llama.cpp](https://github.com/ggml-org/llama.cpp), pinned as a submodule at `vendor/llama.cpp/`. No original C/C++ lives here — only `.ps1` scripts driving CMake + MSVC + Conda, plus one benchmark helper in `examples/mtp-bench.py`. The session shell is bash on Windows; the project's own scripts must run from `pwsh`/`powershell`. ## Commands @@ -21,261 +21,60 @@ Get-Help -Detailed ./examples/server.ps1 # full option list Binaries land in `./vendor/llama.cpp/build/bin/Release/`. Conda env `llama.cpp` (Python 3.12) must already exist — the scripts call `conda activate llama.cpp` themselves. +Other helpers in `examples/`: `count_tokens.ps1`, `benchmark.ps1` (perplexity), `speculative_decoding.ps1`, `speed-bench.ps1` (router-mode throughput sweep), `mtp-bench.py`. `README.md` -> *Usage* is the end-user inventory of what they do; don't restate their flags here. `Get-Help` works on `server.ps1`, `speed-bench.ps1` and `count_tokens.ps1` only — the other two carry no comment-based help. + **No tests, no linter.** Verify changes by running an example script against a real GGUF model. ## Non-obvious behavior -- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt` (idempotent; workaround for `find_package(BLAS)` failing on Windows). `.gitmodules` sets `ignore = dirty` for this reason — don't "clean it up." -- **Each build wipes the submodule** back to `origin/master` then checks out the requested `-version` / PR. Any local edits under `vendor/llama.cpp/` are lost by design. The reset/`--remote` step is **scoped to `vendor/llama.cpp` only** — other submodules (e.g. `vendor/Qwen-Fixed-Chat-Templates`, default branch `main`) stay at the SHA pinned in the superproject and are never advanced by the build script. To bump them, do it manually: `git -C vendor/Qwen-Fixed-Chat-Templates fetch && git -C vendor/Qwen-Fixed-Chat-Templates checkout && git add vendor/Qwen-Fixed-Chat-Templates && git commit`. Once the pin is committed, the next `rebuild_llama.cpp.ps1` mirrors it into the working tree (auto-discovered from `.gitmodules`, `--force`); hand-edits inside the submodule do not survive a rebuild. -- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`.** Upstream `ggml/CMakeLists.txt` sets `cmake_policy(SET CMP0194 NEW)` and declares `project(... ASM)`; on CMake 4.1+ with the VS generator this rejects `cl.exe` as the ASM compiler. The script locates `ml64.exe` via `vswhere.exe`. Don't remove. The `vswhere` call passes `-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64` alongside `-latest` — this is deliberate, not redundant: `-latest` alone returns the newest-installed instance by timestamp, which on a machine with multiple instances may be a Build Tools install lacking the C++ workload, so `-find` returns nothing and the build throws "ml64.exe not found" even though another instance (e.g. Community) has it (#3). `-requires` narrows `-latest` to instances that actually carry the MSVC x64 toolset, matching the pattern upstream uses in `.github/workflows/build-cpu.yml`. Don't drop the `-requires` filter. +- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an idempotent OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt`; `.gitmodules` sets `ignore = dirty` for it. Don't "clean it up." `docs/build_system.md` -> *Submodule lifecycle* +- **Each build wipes `vendor/llama.cpp`** back to `origin/master` then checks out the requested `-version` / PR, so local edits there are lost by design. Other submodules are never advanced by the build script and must be bumped by hand. `docs/build_system.md` -> *Submodule lifecycle* +- **`./patches/*.patch` is the only supported way to carry a local change into the submodule.** It is re-applied after every checkout and the build aborts if a patch no longer applies. Currently one patch, which keeps GGUF conversion from stalling when a tensor's row width puts the quantizer group on a slow allocation size. `docs/build_system.md` -> *Local patches* - **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. -- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`.** Without it the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and the dispatcher returns `BEST_FATTN_KERNEL_NONE` -> `GGML_ABORT` for any other K type or any mismatched K/V pair (`ggml-cuda/fattn.cu:424-446`). The flag pulls in the full `fattn-vec*.cu` set so asymmetric / q5 / q4_1 caches work (the presets use `q5_0` K + `q4_1` V). Costs extra `nvcc` compile time. -- **Build parallelism is SMT-aware.** `cmake --build --parallel` is fed a count derived from `Win32_Processor`. Upstream's `UseMultiToolTask=true` + `EnforceProcessCountAcrossBuilds=true` (`vendor/llama.cpp/CMakeLists.txt:92-93`) makes this the single cap on concurrent `cl.exe`/`nvcc` — no per-project `/MP` multiplication. On SMT CPUs it uses physical cores (`Sum(NumberOfCores)`): dropping the logical siblings avoids starving the scheduler / ~doubling peak `nvcc` RAM (no throughput gain) and leaves them free so the machine stays usable. On non-SMT CPUs where physical == logical (hybrid Arrow/Lunar Lake; e.g. Core Ultra 9 285HX = 8P+16E, 24 threads) using all cores would peg the box at 100%, so it backs off to 80% of physical (`floor(cores * 0.8)` = 19 on the 285HX) to keep the machine usable during builds. Override with `-parallelJobs N`. -- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`.** It pins `torch` to a CUDA 12.6 wheel, adds `tiktoken` (missing upstream, required for GLM), pins `transformers==5.3.0`, and narrows `numpy` to resolve an `opencv-python-headless` conflict. When bumping any of these, verify both constraints still hold. -- **`server.ps1` reads GGUF metadata** by shelling out to `vendor/llama.cpp/gguf-py/gguf/scripts/gguf_dump.py`. Upstream has moved this path before (CHANGELOG 1.24.0) — if server startup fails with "Failed to extract model details", check the path first. +- **Three vendored paths are hardcoded** (`gguf_dump.py`, `speed-bench/`, `models/templates/`). Upstream has moved them before; after a version bump treat a startup failure naming one as a relocation first. `docs/build_system.md` -> *Upstream path dependencies* - **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. -- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. If startup fails reading the script after a rebuild, check whether upstream moved `tools/server/bench/speed-bench/` (same failure mode as the `gguf_dump.py` note above). - **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. -- **`load-mode = dio` does not enable DirectIO on Windows — it only disables mmap.** The Win32 `llama_file::impl` ctor takes `use_direct_io` as `[[maybe_unused]]` and just calls `ggml_fopen` (`vendor/llama.cpp/src/llama-mmap.cpp:86-95`); `FILE_FLAG_NO_BUFFERING` is never set and `read_alignment()` stays 1 (`:391`), so the loader's async staging buffers are 4 x 1 MiB of pinned host memory instead of the 4 x 64 MiB the aligned path would use (`src/llama-model-loader.cpp:1418`, `:1427`). `has_direct_io()` nevertheless returns a hardcoded `true` on Windows (`:173-175`). Net effect of `dio` on this platform: buffered reads, no mmap, and zero VRAM cost — it is never implicated in a CUDA OOM. Keep the key for the deprecation-warning reason documented under Presets, but do not reason about page-cache behaviour from it. ## Presets -VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, -`presets/models_16GB_8GB_VRAM.ini` (dual-GPU). -See `presets/README.md` for the user-facing quick-start; notes below are for editing. - -- **Only `models_16GB_8GB_VRAM.ini` pins devices.** It sets `split-mode`, `main-gpu`, and - `tensor-split` in its `[*]` section; the other two tiers set none of the three, so on a host with - more than one CUDA device llama.cpp's default `split-mode = layer` spreads every entry across - *all* visible GPUs — the tier name is then a floor, not a cap. Pin with `CUDA_VISIBLE_DEVICES` - before launch, not `--device`: in router mode each child's argv is rebuilt from the preset - (`inst.meta.update_args`), so a parent `--device` never reaches the child, while the environment - is copied into every child (`tools/server/server-models.cpp:802`). Pinning is a throughput - decision, not only a memory one — `Ternary-Bonsai-27B-Q2_g64.gguf` measured 56.5 t/s tg and - 1418 t/s pp on one 16 GB card versus 36.5 t/s and 998 t/s spread over a 16 GB plus an 8 GB card. - `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and - therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. - -- **All entries use `load-mode = dio`; never pair `direct-io` with `no-mmap` again.** Both spellings - are deprecated, and they write the *same* mutually exclusive enum — `--no-mmap` sets - `LLAMA_LOAD_MODE_NONE` (`common/arg.cpp:2594`) while `--direct-io` sets - `LLAMA_LOAD_MODE_DIRECT_IO` (`:2603`) — so setting both means only whichever is parsed last wins. - Which one that is is *not* the INI order: `common/preset.cpp` emits `opt.args.back()` while - iterating an unordered map. It happened to resolve to DirectIO, but a container-order change would - silently downgrade to `NONE`, dropping DirectIO *and* mmap for plain buffered reads. `dio` is the - single equivalent of the old pair and also removes two deprecation warnings per launch. Valid - values are `none`, `mmap`, `mlock`, `mmap+mlock`, `dio` (`arg.cpp:2615-2619`) — anything else - throws at startup. - -- **Qwen-VL entries pin `image-min-tokens = 1024`.** `clip.cpp:1500` sets the per-image token limits - to `(8, 4096)`, so the default *minimum* is 8 tokens — 8192 px at `merge = 2` / `patch = 16` — and - `clip.cpp:1502-1506` warns on every load because upstream needs >= 1024 tokens (1024x1024 px) for - grounding (#16842). The key raises a floor only: images already above 1024 tokens are unchanged, - smaller ones get upscaled, which costs context and CLIP time on the CPU because the 16 GB tier - sets `no-mmproj-offload = true`. Applies to the `qwen3vl_merger` entries (Qwen3.6 and - Ternary-Bonsai); gemma-4 uses a different projector and must not get this key. - -- **`mmproj-offload = true` fails silently at startup on a saturated GPU.** CLIP's warmup - compute buffer OOMs but the server keeps running — only image requests error at generation - time. Set `false` on tiers where LLM + KV already saturate VRAM. - -- **All Qwen 3.6 entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** - Required, *not* redundant with `jinja = true` — `chat-template-file` *replaces* - the GGUF-embedded template entirely (`vendor/llama.cpp/common/arg.cpp:3142`, - `params.chat_template = read_file(value)`). The upstream embedded template has - documented issues with tool calls, role handling, `` block rendering, - agentic loops, and llama.cpp KV-prefix cache stability; the vendored template - fixes all of them (full list in `vendor/Qwen-Fixed-Chat-Templates/README.md`). - Since v19 the template is a single unified file covering both Qwen 3.5 and 3.6 - variants (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under - `archive/`). The template adds a `<|think_on|>` / `<|think_off|>` toggle, and - v19 defaults `preserve_thinking` to `true` (past `` blocks are kept - chronologically for 100% KV prefix cache stability and agentic reasoning - continuity). To strip past `` blocks instead, set - `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower - KV cache hit rate. Path is repo-relative, so `llama-server` must be launched - from the repo root — `read_file()` resolves against the process CWD, not the - INI file's directory. `Qwen3-Coder-Next` entries deliberately keep their - GGUF-embedded template; froggeric's README only claims compatibility for - Qwen 3.5 / 3.6 variants. - -- **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** - This is Google's fixed official template as aligned by upstream (#21704) — the exact - file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler - (`vendor/llama.cpp/common/chat.cpp:1216`), so parser and template always come from the - same submodule commit (each rebuild resets the submodule to master, mirroring the built - binary). GGUF-embedded templates from conversions predating Google's template fixes - lack the `{#- OpenAI Chat Completions:` marker; llama.cpp then logs "detected an - outdated gemma4 chat template" and rewrites messages via C++ compatibility workarounds - (`common/chat.cpp:2250-2258`) — the pin avoids that path. One file covers the whole - series (12B / 26B-A4B / 31B, incl. `<|image|>`/`<|audio|>` placeholders), and - `reasoning = on` maps to the template's `enable_thinking` kwarg - (`common/arg.cpp:3167-3175`), so no `chat-template-kwargs` are needed. Unlike the Qwen - template, past `<|channel>thought` blocks are *stripped* from history by design — - Gemma 4 is trained that way — so cross-turn KV-prefix invalidation is inherent - (`ctx-checkpoints` mitigates); do not add a preserve-thinking hack. If startup fails - reading the template after a rebuild, check whether upstream moved - `models/templates/` (same failure mode as the `gguf_dump.py` note above). - -- **Both Bonsai entries use the same `chat-template-file` pin as the Qwen 3.6 entries.** - `Ternary-Bonsai-27B` and `Bonsai-27B` ship from separate HF repos but are both Qwen3.6-27B - derivatives: arch `qwen35`, and their tokenizers are byte-identical to stock Qwen3.6-27B - (248320 tokens, same merges, `eos = 248046`) right down to the same 7764-byte embedded template — - which is exactly the upstream template the pin exists to replace. `general.sampling.temp = 1.0` is - embedded in both GGUFs and applied at `common/common.cpp:1194`, so `temp` has to be pinned in the - preset or generation runs at 1.0. The presets use `0.6` to match the sibling Qwen 3.6 entries; - Prism's own card benchmarks at `0.7`. Unlike the DSpark sidecar below, both weight files are - mainline-packed (`Q2_0` at `QK2_0 64`, `Q1_0` at `QK1_0 128`) and load without a tensor-offset - mismatch. - -- **The DSpark drafter shipped beside Ternary Bonsai 27B cannot be enabled on mainline.** - `Ternary-Bonsai-27B-dspark-Q4_1.gguf` has its `token_embd.weight` in `Q2_0` at Prism's - group-128 packing while mainline is group-64 (`QK2_0 64`, `ggml/src/ggml-common.h`), so - `gguf_init_from_reader` rejects the file on a tensor-offset mismatch before any architecture - dispatch. Repacking would not help: `general.architecture = 'dspark'` is unregistered - (`src/llama-arch.cpp:136` has only `dflash`), and mainline's DSpark is DeepSeek-V4 - DFlash + Markov (`src/models/dflash.cpp`, tensors `markov_w1`/`markov_w2`/`conf_proj`, - requiring MLA and sqrtsoftplus MoE scoring), not Prism's 6-layer Qwen3.6-shaped drafter - (`dspark.fc`, `dspark.log_snr_fc*`, `dspark.markov_head_*`). Upstream confirmed on #25707 - that it stays fork-only. The GGUF carries no MTP tensors either, so `draft-mtp` is out and - the entry uses `ngram-mod`. Only the group-64 pack is mainline-loadable — `Q2_0.gguf` and - `PQ2_0.gguf` in the same HF repo are group-128 fork packs. +VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, `presets/models_16GB_8GB_VRAM.ini` (dual-GPU). -- **DeepSeek-V4-Flash-0731 must set `cache-type-k` *and* `cache-type-v` to the same value.** - Arch is `deepseek4` (the HF card's "dflash / 20B" is the DSpark sidecar's metadata, not the - model). `llama-context.cpp:3560-3563` compares the two values and refuses to create the - context — `does not support different K (%s) and V (%s) cache types` — for - `LLM_ARCH_DEEPSEEK4` specifically, because `hparams.is_mla()` is *false* for this arch and the - guard needs the explicit disjunct. So `cache-type-v = q8_0` is load-bearing even though V is - never allocated: DSV4 is K-only everywhere (`dsv4_make_k_only()` at - `llama-kv-cache-dsv4.cpp:831-835` forces `is_mla` true on hparams copies, so - `has_v = !is_mla` at `llama-kv-cache.cpp:229` is false). Copying the Qwen dual-GPU pair - `q5_0` K / `q4_1` V here is startup-fatal, not merely wasteful. `q8_0` also clears - `n_embd_head_k() % 64 == 0` so quantized K gets the Hadamard rotation - (`llama-kv-cache.cpp:319-323`); the lightning-indexer cache is rotated unconditionally for - this arch (`:325-329`). `kv-unified` is silently discarded (`GGML_UNUSED(unified)`, - `dsv4.cpp:1189-1192`), and `cache-type-*-draft` is dead without a draft model. +`presets/README.md` is the user-facing quick-start. For editing, the cross-model rules are in `docs/presets.md` and the per-model rationale with its measured numbers is in `docs/model_tuning.md`. -- **The DSV4 KV cache is tiny, so context is cheap and quantizing it buys little.** 43 layers - split 2 raw / 21 CSA (ratio 4) / 20 HCA (ratio 128) via `attention.compress_ratios`, all - K-only at `n_embd_k_gqa = 512`; the raw tier is SWA-windowed to - `PAD(min(n_ctx, 128 + n_ubatch), 256)` = 768 cells regardless of `ctx-size`. At 262144 that - is 942 MiB at `q8_0` (1764 MiB at f16) plus a fixed 11.64 MiB of F32 compressor state that no - cache type shrinks. Never set `swa-full`: it collapses the window formula to `n_ctx` - (`llama-kv-cache-iswa.cpp:76-81`), turning a 17 MiB raw cache into ~11 GiB. The server warns - `swa_full is not supported` only *after* the cache is built, so the flag still takes effect. +## Traps -- **Leave `fit = on` and never add `n-cpu-moe`/`-ot` to the DeepSeek entry.** Measured UD-Q8_K_XL - composition: 137.06 GiB routed experts (MXFP4, 90.9%), 2.02 GiB shared experts, 11.67 GiB - non-expert — so non-expert + shared is only 13.69 GiB and fits a 24 GB card alongside the KV - with room for a full expert layer or two (3.19 GiB each). `fit` finds that split at sub-layer - granularity (`fit.cpp:399-441`, `:719-769`); forcing `-ncmoe 43` would push all experts to CPU - and strand ~9 GiB of VRAM. Any user `-ot`/`--cpu-moe`/`--n-cpu-moe` aborts fit outright - (`fit.cpp:395-397`), as does setting `n-gpu-layers` to anything but `-1` (`fit.cpp:374-376`) — - which is why the entry keeps `-1` explicitly. Note `--cpu-moe`'s pattern matches only - `_exps`/`_chexps`, so shared experts would stay on GPU either way. +Prohibitions that cause a silent OOM, silent corruption, or a startup abort. Each is stated here without rationale so it is always in context; read the linked section before acting on one. These lines are a deliberate projection of `docs/` — when a trap changes, both move together. -- **`no-host = true` is mandatory on the DeepSeek entry, and this is the trap that actually stops - it loading.** Unless `no_host` is set, `make_cpu_buft_list()` prepends - `ggml_backend_dev_host_buffer_type()` to the CPU buffer list, so *every* CPU-resident tensor is - allocated in a `CUDA_Host` (page-locked) buffer (`src/llama-model.cpp:896-917`, wired from - `params.no_host` at `common/common.cpp:1611` and `include/llama.h:338`). For this model the - loader then reports one `CUDA_Host model buffer size = 137046.96 MiB` — a 133.8 GiB - `cudaMallocHost` on a 192 GB box. The reservation *succeeds*, so `ggml_cuda_host_malloc`'s - clean-failure fallback to an ordinary CPU buffer never fires; the failure happens later while - the pages are committed during the read and surfaces as `CUDA error: out of memory` inside - `cudaEventSynchronize` at `src/llama-model-loader.cpp:1591`. That makes a host-memory problem - look like a VRAM problem — raising `fit-target` does not help it, and neither does changing - `load-mode`. With `no-host = true` the same config loads in ~144 s at 18650 MiB VRAM and - ~124 GiB of ordinary host RAM. `GGML_CUDA_NO_PINNED=1` is the env-var equivalent. This applies - to any entry that pushes tens of GiB of experts to CPU, not only DeepSeek. The GPU upload - staging buffers are unaffected — the loader asks for those buffer types directly - (`src/llama-model-loader.cpp:1467`) rather than through `cpu_buft_list` — but expert weights - that `op-offload` ships to the GPU for large-batch matmuls now come from pageable memory, which - may cost some prompt-processing throughput. There is no way to keep that and still load. - -- **`fit-target = 3072` on the DeepSeek entry is a WDDM safety margin, not the fix for the load - failure** (that is `no-host` above). `fit` measures rather than guesses — it performs a - `no_alloc` model load plus a real graph reservation (`fit.cpp:56-75`), so its KV figure is - byte-exact (942 MiB at 262144/`q8_0`) and its compute figure is a genuine `ggml_gallocr` - measurement. What it cannot see is the CUDA VMM scratch pool (32 GiB of VA reserved, physical - pages committed on demand, `ggml/src/ggml-cuda/ggml-cuda.cu:536-656`), the lazy cuBLAS - workspace, and CUDA graph instances; none are reported to `memory_breakdown()`. It also takes a - single `cudaMemGetInfo` snapshot at t=0 (`fit.cpp:194`) and carries no WDDM or framebuffer - allowance anywhere. At the default 1024 MiB margin fit keeps blk.0 and blk.1 routed experts on - the GPU (6.375 GiB — every one of the 43 layers carries 3.188 GiB of routed experts, there are - no dense layers) and leaves only 1368 of 23139 usable MiB for those untracked consumers. - `3072` leaves 3497 MiB and costs one extra expert layer on CPU (~2.3% more expert traffic). - Do not raise it to 6144: that collapses `-ngl` to 38 and starts stranding whole layers. - `--fit-target` writes only `params.fit_params_target` and never `mparams`, so unlike - `-ngl`/`-ncmoe`/`-ot` it cannot trip the aborts at `fit.cpp:374-397`. - -- **`cache-ram` is 16384 on the DeepSeek entry, not the 51200 used elsewhere.** `fit` reports - 133.8 GiB of `Host model` weights for this entry (measured at `fit-target = 3072`), so on a - 192 GB box a 50 GiB prompt cache overcommits and pages. A full-context prompt - state at 262144/`q8_0` is 931 MiB (`server-task.cpp:1671-1683` — an entry larger than the - whole limit is silently skipped), so 16 GiB still holds ~17 of them. Context checkpoints are - separate and cheap: 14.5 MiB each and independent of `ctx-size`, because a DSV4 checkpoint - stores only the 128-position SWA window plus the fixed compressor state. - -- **The DeepSeek-V4-Flash entry deliberately does *not* pin `chat-template-file`.** This is the - one exception to the convention above. The GGUF ships Unsloth's fixed template, and both it - and upstream's bundled `models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja` (#26398) - satisfy the detection heuristic at `common/chat.cpp:3170-3179` (`dsml_token` + `DSML` + - `tool_calls`), so both route to the native PEG parser - (`common_chat_params_init_deepseek_v3_2`, `chat.cpp:2097`) and classify as V4 via the - `function_calls`-absent test at `:2105`. Unsloth's additionally restores `reasoning_content` - on tool calls, which the official template drops. Unlike gemma-4 there is no outdated-template - rewrite path for `deepseek4` — detection is all-or-nothing, and a miss degrades to the generic - autoparser rather than being repaired. `reasoning_effort` has no CLI flag - (`server-common.cpp:1089-1095` honours only the literal `"none"`), so the only route is - `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); left unset the template - defaults it to `none` and emits no effort block at all, and `reasoning = off` voids it - entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` - (`common.h:631`, despite the help text saying `auto`), and `none` leaks `` into - `content`. - -- **Context shift and cache-reuse are permanently unavailable on `deepseek4`.** - `llama_kv_cache_dsv4::get_can_shift()` returns false (`dsv4.cpp:1394-1398`), so the server - force-disables both with a warning (`server-context.cpp:1268-1278`); slots then stop cleanly - at `STOP_TYPE_LIMIT` instead of shifting. `seq_rm` also refuses partial removal when - `n_rs_seq == 0` (`dsv4.cpp:1427-1429`), which `ngram-mod` does not set, so rollback goes - through checkpoints — correct, but each rejected draft costs a ~14.5 MiB state restore and the - net throughput effect is uncharacterised. The `dspark` sidecar in the same HF repo is a - genuine mainline `dflash` drafter (unlike the Bonsai one above), but is unusable here: its - README requires `--fit off` plus full offload of target *and* drafter (11 GiB drafter + - 13.69 GiB non-expert exceeds 24 GB), `--spec-draft-n-max` is clamped to 5, multi-GPU needs a - rebuild with `GGML_SCHED_MAX_SPLIT_INPUTS=48`, and it carries an open decode-time CUDA abort - after ~2500 tokens (#26554). The regression that broke spec decoding on this arch (#26576, - a 2D `wo_a` in `dflash.cpp` after #26531) is fixed by #26577 at `b10269`. - -**ngram-mod speculative decoding** (`--spec-type ngram-mod`): model-agnostic, works on any model. -- All models: `spec-ngram-mod-n-match = 24`, `spec-ngram-mod-n-min = 48`, `spec-ngram-mod-n-max = 64` - (matches the struct defaults in `common/common.h:329-337` and what `--spec-default` produces - at `common/arg.cpp:4065-4074`; ggerganov confirmed post-merge in PR #19164 that the min/max - "likely don't need to be changed from the recommended values"; MoEs require long drafts and - dense models tolerate them without noticeable cost). Flags were renamed from - `--draft-min`/`--draft-max`/`--spec-ngram-size-n` in upstream PR #22397; the old names now - error at startup. -- `n_match < 16` logs a "too small — poor quality is possible" warning at - `vendor/llama.cpp/common/speculative.cpp:1031-1034`; parser accepts `1..1024` - (`common/arg.cpp:3606-3615`), so 16 is the lowest non-warning value, not a hard floor. - Min/max parsers accept `0..1024` (`common/arg.cpp:3587-3605`). -- Memory overhead: ~16 MiB **total**, shared across all server slots - (single `common_ngram_mod` instance allocated at `common/speculative.cpp:1026`). -- Pool auto-resets on `begin()` if occupancy > 25 %, and after 3 consecutive rounds with - acceptance < 50 % (`common/speculative.cpp:720-728`, `:790-806`). Smaller `n_match` makes - these resets fire more often and wipes ngrams learned from the current prompt — another - reason to stay at `n_match ≥ 24`. +- Never pair `direct-io` with `no-mmap`; use `load-mode = dio` — except on `Qwen3.8-Flash-Next`, which must keep `load-mode = mmap`. `docs/presets.md` -> *load-mode* +- Never set `mmproj-offload = true` on a tier where LLM + KV already saturate VRAM. `docs/presets.md` -> *mmproj-offload* +- Never set `swa-full` on a DeepSeek-V4-Flash or Muse Glimmer entry. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Muse Glimmer* +- Never set `context-shift` on a Muse Glimmer entry; it is silent corruption, not a refusal. `docs/model_tuning.md` -> *Muse Glimmer* +- Never add RoPE scaling to a Muse Glimmer entry. `docs/model_tuning.md` -> *Muse Glimmer* +- `no-host = true` is mandatory on the DeepSeek and `Qwen3.8-Flash-Next` entries, and on any entry that pushes tens of GiB of experts to CPU; without it the load fails as a misleading CUDA OOM. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Qwen3.8-Flash-Next* +- Keep `fit = on` on the DeepSeek and `Qwen3.8-Flash-Next` entries; never add `n-cpu-moe`/`-ot`, and never set `n-gpu-layers` to anything but `-1` — fit then silently no-ops. `docs/model_tuning.md` -> *DeepSeek-V4-Flash*, *Qwen3.8-Flash-Next* +- `cache-type-k` and `cache-type-v` must be identical on `deepseek4`; differing values are startup-fatal. `docs/model_tuning.md` -> *DeepSeek-V4-Flash* +- Never set `image-min-tokens` on a gemma-4 entry; it is a `qwen3vl_merger` key only. `docs/model_tuning.md` -> *Qwen 3.6 and 3.8* +- Never drop a `chat-template-file` pin; it replaces the GGUF-embedded template and is not redundant with `jinja = true`. `docs/model_tuning.md` +- Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from the derived `-FP8` repo. `docs/model_tuning.md` -> *Qwen 3.6 and 3.8* ## Changelog style - One bullet = one physical line. Never insert manual line-breaks; let the editor soft-wrap. - Format: `- [Component] ` (Added / Changed / Fixed / Removed). -- No rationale, no file paths, no line numbers, no explanatory prose. Rationale lives in AGENTS.md "Non-obvious behavior" or in the commit message. +- No rationale, no file paths, no line numbers, no explanatory prose. Rationale lives in AGENTS.md "Non-obvious behavior", the matching `docs/` file, or the commit message. - PR refs as bare `#NNNNN`, at most once per release. - Canonical examples: [1.21.0] – [1.27.0] in CHANGELOG.md. ## Scratch Files -Non-committed agent artifacts (diffs, trace outputs, generated reports, experimental scripts) go under `.tmp/sessions//` at the repo root; `.tmp/` is gitignored. `` is `SESSION_ID` when the platform injects it, otherwise a minted `YYYYMMDD-HHMMSS-`. Never write scratch files to `.claude/`, the repo root, or `vendor/`. +Non-committed agent artifacts (diffs, trace outputs, generated reports, experimental scripts) go under `.tmp/sessions//` at the repo root; `.tmp/` is gitignored. `` is the `SESSION_ID` injected into context at session start — by the `SessionStart` hook in `.claude/settings.json` under Claude Code, by `.opencode/plugins/session-id-injector.js` under OpenCode. If neither fired and no `SESSION_ID` is in context, mint `YYYYMMDD-HHMMSS-` instead. Never write scratch files to `.claude/`, the repo root, or `vendor/`. + +## Reference + +Deep reference documentation lives under `docs/` and is **read on demand**, not loaded into context. Consult the relevant file when a task touches its area: + +| Document | When to read | +| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs/build_system.md` | Why the build scripts do what they do: submodule lifecycle, `ml64.exe`/`vswhere` toolchain detection, CUDA flags, SMT-aware parallelism, the Python requirements layering, the hardcoded upstream paths, and how `speed-bench.ps1` drives a router-mode server. **Read before editing `rebuild_llama.cpp.ps1` or any `examples/*.ps1`, and before running `speed-bench.ps1`.** | +| `docs/presets.md` | Cross-model INI rules: device pinning and multi-GPU, `load-mode`, `mmproj-offload`, context size and `override-kv`, ngram-mod speculative decoding. **Read before editing any file under `presets/`.** | +| `docs/model_tuning.md` | Per-family rationale and measured VRAM/throughput numbers for Qwen 3.6 and 3.8, gemma-4, Bonsai and DSpark, DeepSeek-V4-Flash, and Muse Glimmer. **Read before adding, retuning or removing a model entry.** | diff --git a/CHANGELOG.md b/CHANGELOG.md index a045649..e09e42d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,73 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.41.0] - 2026-08-28 + +### Added +- [Presets] Add Qwen3.8-27B IQ4_XS entry to the 24 GB tier +- [Presets] Add Qwen3.8-Flash-Next IQ4_XS entry to the 24 GB tier +- [Presets] Pin chat-template-file on the Qwen3.8-27B entry +- [Presets] Pin reasoning-effort to medium on all Qwen 3.6 and Bonsai entries +- [Presets] Pin reasoning-effort to xhigh on the Qwen3.8 entries +- [Documentation] Document why Qwen3.8-27B now pins the vendored chat template +- [Documentation] Document the reasoning-effort steering and the default-effort knob in the vendored chat template +- [Documentation] Document the reasoning-effort aliases the vendored template maps for OpenAI, Claude Code, Cursor and Cline +- [Documentation] Document the tools serialization and think-tag spacing fixes in the vendored chat template +- [Documentation] Document the two v19 prompt-format deviations the vendored template corrects +- [Documentation] Document the empty think block the vendored template emits before a tool call that carries no reasoning +- [Documentation] Document that pinning reasoning-effort is what keeps a template bump from moving the level +- [Documentation] Document why Qwen3.8-27B keeps xhigh reasoning and how reasoning-budget bounds it +- [Documentation] Document the standalone Qwen3.8-27B MTP sidecars as an alternative to a re-quant +- [Documentation] Document the GGML_SCHED_MAX_COPIES=1 CUDA build flag +- [Documentation] Document the Qwen3.8-27B projector choice and its quantization source +- [Documentation] Document that ctx-size above the GGUF context_length is dead VRAM without override-kv +- [Documentation] Document Muse-Glimmer-30B RoPE, swa-full, and context-shift behavior +- [Documentation] Document the Muse-Glimmer-30B DFlash drafter, chat template, and reasoning strength defaults +- [Documentation] Document the measured Muse-Glimmer-30B 24 GB compute buffer and VRAM breakdown +- [Documentation] Add an always-on trap index and a docs reference table to AGENTS.md +- [Agents] Add session-retrospective skill for routing session learnings into AGENTS.md and docs/ +- [Agents] Inject SESSION_ID at session start for Claude Code and OpenCode +- [Build] Apply patches from ./patches to the llama.cpp submodule after checkout +- [Build] Abort the build when a local patch no longer applies +- [Documentation] Document the local patch mechanism in the build system reference +- [Documentation] Document the gguf-py row-group fix, the platforms it was measured on, and the three variants that were rejected +- [Documentation] Document the Qwen3.8-Flash-Next per_layer_token_embd table and its lazy read path +- [Documentation] Document why the Qwen3.8-Flash-Next entry needs load-mode = mmap instead of dio +- [Documentation] Document why no-host keeps the Qwen3.8-Flash-Next mmap aliasing path reachable +- [Documentation] Document the Qwen3.8-Flash-Next weight split and the two fit blind spots +- [Documentation] Document the Qwen3.8-Flash-Next QSA indexer cache and its unused V allocation +- [Documentation] Document that Qwen3.8-Flash-Next has no MTP head and cannot shift context or reuse cache +- [Documentation] Document the Qwen3.8-Flash-Next recurrent state cost and why parallel is the cheap knob +- [Documentation] Document that a Qwen3.8-Flash-Next context checkpoint is the whole recurrent state +- [Documentation] Document the measured Qwen3.8-Flash-Next 24 GB VRAM and throughput figures +- [Documentation] Document the measured Qwen3.8-Flash-Next context and compute cost per n_ctx_seq +- [Documentation] Document why a 1M cell pool does not fit Qwen3.8-Flash-Next on 24 GB + +### Changed +- [Documentation] Move the preset rationale out of AGENTS.md into docs/presets.md and docs/model_tuning.md +- [Documentation] Move the build-system rationale out of AGENTS.md into docs/build_system.md +- [Documentation] Drop the build-configuration summaries from AGENTS.md that docs/build_system.md already carries +- [Documentation] List the remaining examples scripts and their help coverage in AGENTS.md +- [Documentation] Extend the no-host and fit traps in AGENTS.md to Qwen3.8-Flash-Next +- [Vendor] Bump Qwen-Fixed-Chat-Templates submodule to v22.4 +- [Vendor] Bump llama.cpp submodule to b10660 for Qwen3.8-Flash-Next support #27742 +- [Presets] Override the Muse-Glimmer-30B context_length metadata to 262144 +- [Presets] Raise Muse-Glimmer-30B 24 GB ctx-size to 524288 and parallel to 2 +- [Presets] Switch Muse-Glimmer-30B 24 GB KV cache to q5_0 K / q4_1 V + +### Removed +- [Presets] Drop the Qwen3.6-27B-uncensored-heretic-v2 entry from the 24 GB tier +- [Agents] Stop tracking .claude/settings.local.json + +### Fixed +- [Documentation] Correct the common.cpp line reference for the embedded sampling temperature +- [Documentation] Correct the claim that reasoning_effort has no CLI flag +- [Documentation] Correct the GGML_CUDA_FA_ALL_QUANTS failure mode from an abort to a silent CPU fallback +- [Documentation] Correct the claim that Muse-Glimmer-30B has no bundled chat template +- [Documentation] Refresh stale llama.cpp line references in the Qwen and Muse Glimmer sections +- [Conversion] Fix GGUF conversion stalling when a tensor row width puts the quantizer group on a slow allocation size + + ## [1.40.0] - 2026-08-05 ### Added diff --git a/docs/build_system.md b/docs/build_system.md new file mode 100644 index 0000000..ba01140 --- /dev/null +++ b/docs/build_system.md @@ -0,0 +1,71 @@ +# Build System Reference + +Why `rebuild_llama.cpp.ps1` and the `examples/*.ps1` scripts do what they do. Not +auto-loaded into the agent context; read it on demand. The one-line rules these +sections back are in AGENTS.md "Non-obvious behavior". + +## Submodule lifecycle + +- **The submodule always shows dirty.** `rebuild_llama.cpp.ps1` prepends an OpenBLAS linking shim to `vendor/llama.cpp/CMakeLists.txt` (idempotent; workaround for `find_package(BLAS)` failing on Windows). `.gitmodules` sets `ignore = dirty` for this reason — don't "clean it up." + +- **Each build wipes the submodule** back to `origin/master` then checks out the requested `-version` / PR. Any local edits under `vendor/llama.cpp/` are lost by design. The reset/`--remote` step is **scoped to `vendor/llama.cpp` only** — other submodules (e.g. `vendor/Qwen-Fixed-Chat-Templates`, default branch `main`) stay at the SHA pinned in the superproject and are never advanced by the build script. To bump them, do it manually: `git -C vendor/Qwen-Fixed-Chat-Templates fetch && git -C vendor/Qwen-Fixed-Chat-Templates checkout && git add vendor/Qwen-Fixed-Chat-Templates && git commit`. Once the pin is committed, the next `rebuild_llama.cpp.ps1` mirrors it into the working tree (auto-discovered from `.gitmodules`, `--force`); hand-edits inside the submodule do not survive a rebuild. + +## Local patches + +- **`./patches/*.patch` are re-applied to `vendor/llama.cpp` on every build.** The step runs after the `-version` / PR checkout (`rebuild_llama.cpp.ps1:254-284`), because the checkout would otherwise discard them. It needs no idempotence guard like the OpenBLAS shim — the `reset --hard` earlier in the script guarantees a clean tree. Applied with `git apply --3way`, so a patch still lands when upstream moves the lines *around* a hunk; if it fails, upstream moved the patched code itself and the script `throw`s rather than silently building an unpatched tree. `--3way` implies `--index`, which would leave the files staged and make `git checkout -- ` restore the *patched* copy instead of the upstream one — the trailing `git reset --quiet` unstages them so they show up as plain worktree modifications, same as the shim. + +- **`0001-gguf-py-size-row-groups-by-bytes.patch` fixes a 27x conversion slowdown.** `_apply_over_grouped_rows` (`gguf-py/gguf/quants.py:29`) split rows into groups of 16 *rows*, so the group's byte size followed the row width. The patch grows the group until it holds 160 KiB of input but never goes below 16 rows, leaving `np.array_split` and `np.concatenate(..., out=out)` untouched. Two lines of logic. Measured full scale (3.355 GB f32 in) on Windows with numpy 2.2.6: a 640-wide row goes **129.4 s -> 4.8 s (27.0x)**; 2560-wide is unchanged (the budget yields exactly 16 rows there, so the patch is a no-op by construction). Across an 18-point width ladder from 128 to 16384 the patched form is flat at ~1.13-1.18 s where upstream ranges 1.12-5.13 s. + +- **The cause is the Windows low-fragmentation heap, and the trigger is the *retained* block, not the working buffer.** Each `func(group)` call allocates several transient temporaries and returns one result the caller retains until the whole tensor is done. Microsoft documents that the LFH does not serve allocations above ~16 KiB. Once the retained result crosses that, the interleaving of long-lived and short-lived blocks fragments the general heap and per-allocation cost grows roughly with the square of the live block count: measured 3.7 us/alloc flat from 1 K to 65 K live blocks at 8 KiB retained, against 13 -> 526 us/alloc over the same range at 32 KiB retained. The cliff was located by varying the result-to-temporary size ratio and watching it move: it sits at 17 KiB retained for a 0.5 ratio and 16.2 KiB for 0.25, pinned to the documented ceiling in both cases. Upstream's 16 rows of 640 f32 lands at 20,480 B retained, just past it. A pure allocator microbenchmark with no gguf code reproduces the whole effect (`.tmp/.../alloc_band.py`), and the same script on glibc is flat at 0.88-1.16x with no band at any size. + +- **The 16-row floor is deliberate and costs Windows real throughput; do not remove it.** Without it the budget asks for fewer than 16 rows once a f32 row exceeds 2560 elements, and that regresses Linux by about 1.2x at widths 4096-8192 — reproduced on WSL1 and on a real Debian 13 / kernel 6.12 / glibc 2.41 VM, twice, at repeats 7 and 15. The floor makes the patch a no-op for every width the budget cannot improve, so it can never be slower than upstream anywhere. The price is paid on Windows: at 5120-wide full scale the floored form is 7.57 s against 4.79 s unfloored (1.58x). That trade was chosen because not regressing other platforms is the harder constraint. Flipping `max(16, ...)` back to `max(1, ...)` buys the Windows win back and is a one-character change. + +- **Three alternatives were measured and rejected.** A larger fixed row count only relocates the fault, because the pathology follows `rows x width x itemsize`: 64 rows fixes 640-wide and breaks 160-wide (4.1 s -> 26.4 s). Writing each group straight into `out` instead of concatenating is faster on Windows with numpy 2.x but regresses 2.2x on numpy 1.26.4, which upstream CI pins, and 2.3x on glibc for wide rows. Bounding the *retained* block to 16 KiB directly, which the mechanism suggests, loses 1.18-1.24x to the 160 KiB input budget at every scale from 1/8 to full. A `np.uint64` widening in `BF16.quantize_blocks` is worth a further 1.41x on Windows / 1.84x on glibc and was measured but **deliberately left out** to keep this patch to one concern; it is a separate change to a line upstream added on purpose in #7843. + +- **It is a throughput fix, not an OOM fix, and the `per_layer_token_embd` figure is per shard.** `conversion/qwen4exp.py:145-172` wraps the 128 PLE shards in `gguf.LazyChunkedTensor`, whose `tofile` quantizes one shard at a time (`gguf-py/gguf/lazy.py:275-289`), so `_apply_over_grouped_rows` only ever sees ~1.6 GB of that table, never all 95 GiB. Peak RSS for it was solved separately by `53c0f624a` (stream the shards, ~300 GB -> one shard) and `a510c82e1` (`LazyChunkedTensor`) — don't re-credit that to this patch. Don't pitch the patch upstream against #15623 / #15648 either: both target peak RSS, and this patch does not change it. + +- **The patch is byte-for-byte equivalent to upstream, and that was verified three ways.** Against the C implementation with `python gguf-py/tests/test_quants.py --libggml build/bin/Release/ggml-base.dll` (25 types, 60 exact matches, 0 mismatches, exit 0); across 10 type and width combinations in both quantize and dequantize directions on Windows/numpy 2.2.6, Windows/numpy 1.26.4 and glibc/numpy 2.2.6; and over 15 adversarial shapes covering a single row, row counts below one group, prime row counts that leave a remainder, and rows up to 262144 wide (180 comparisons per numpy version, 0 failures). Note `test_quants.py` needs `ggml-base.dll`, not `ggml.dll` — only the former exports `ggml_quantize_chunk` — and `ctypes` needs both `%CUDA_PATH%\bin` and `%CUDA_PATH%\bin\x64` added via `os.add_dll_directory` before the load will resolve. + +## Toolchain detection + +- **`ml64.exe` (MASM) must be passed as `-DCMAKE_ASM_COMPILER`.** Upstream `ggml/CMakeLists.txt` sets `cmake_policy(SET CMP0194 NEW)` and declares `project(... ASM)`; on CMake 4.1+ with the VS generator this rejects `cl.exe` as the ASM compiler. The script locates `ml64.exe` via `vswhere.exe`. Don't remove. The `vswhere` call passes `-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64` alongside `-latest` — this is deliberate, not redundant: `-latest` alone returns the newest-installed instance by timestamp, which on a machine with multiple instances may be a Build Tools install lacking the C++ workload, so `-find` returns nothing and the build throws "ml64.exe not found" even though another instance (e.g. Community) has it (#3). `-requires` narrows `-latest` to instances that actually carry the MSVC x64 toolset, matching the pattern upstream uses in `.github/workflows/build-cpu.yml`. Don't drop the `-requires` filter. + +- **CUDA is selected iff *both* `nvidia-smi` and `nvcc` are on PATH.** Missing either silently falls back to OpenBLAS. + +## CUDA build flags + +- **CUDA builds pass `-DGGML_CUDA_FA_ALL_QUANTS=ON`, and the failure without it is silent.** Without the flag the CUDA flash-attention path compiles only four *symmetric* KV kernels — `f16/f16`, `q4_0/q4_0`, `q8_0/q8_0`, `bf16/bf16` (`vendor/llama.cpp/ggml/src/ggml-cuda/CMakeLists.txt:119-124`) — and `ggml_cuda_get_best_fattn_kernel` returns `BEST_FATTN_KERNEL_NONE` for `q4_1` / `q5_0` / `q5_1` (`ggml-cuda/fattn.cu:338-356`) or any mismatched K/V pair (`:442-446`). The `GGML_ABORT` at `fattn.cu:574` is unreachable: `supports_op` consults `ggml_cuda_flash_attn_ext_supported` first (`ggml-cuda/ggml-cuda.cu:5287`), so the scheduler simply places `FLASH_ATTN_EXT` on the **CPU** backend — no error, no log line, a ~20-30x prefill collapse (#27109). Both Muse Glimmer entries use `q5_0` K + `q4_1` V and would hit it. The flag pulls in the full `fattn-vec*.cu` set at the cost of extra `nvcc` compile time. + +- **CUDA builds pass `-DGGML_SCHED_MAX_COPIES=1` (ggml's default is 4).** `sched->n_copies = parallel ? GGML_SCHED_MAX_COPIES : 1` (`vendor/llama.cpp/ggml/src/ggml-backend.cpp:1804`), so pinning it to 1 leaves every multi-copy staging branch dead even when `cparams.pipeline_parallel` is true (`src/llama-context.cpp:428-455`). Single-stream decode gains nothing from the extra copies and they cost ~1.6 GiB on a two-device layer split. It is also what keeps this repo out of #26873, where with pipeline parallelism active one image request permanently costs ~39% prefill for the process lifetime — reproduced on Qwen3.8-27B + mmproj and on Muse Glimmer, with `-ot zzz_never_matches=CUDA0` (which disables pipeline parallelism outright) the only workaround found upstream. + +## Build parallelism + +- **Build parallelism is SMT-aware.** `cmake --build --parallel` is fed a count derived from `Win32_Processor`. Upstream's `UseMultiToolTask=true` + `EnforceProcessCountAcrossBuilds=true` (`vendor/llama.cpp/CMakeLists.txt:92-93`) makes this the single cap on concurrent `cl.exe`/`nvcc` — no per-project `/MP` multiplication. On SMT CPUs it uses physical cores (`Sum(NumberOfCores)`): dropping the logical siblings avoids starving the scheduler / ~doubling peak `nvcc` RAM (no throughput gain) and leaves them free so the machine stays usable. On non-SMT CPUs where physical == logical (hybrid Arrow/Lunar Lake; e.g. Core Ultra 9 285HX = 8P+16E, 24 threads) using all cores would peg the box at 100%, so it backs off to 80% of physical (`floor(cores * 0.8)` = 19 on the 285HX) to keep the machine usable during builds. Override with `-parallelJobs N`. + +## Python requirements layering + +- **`requirements_override.txt` layers on top of upstream `vendor/llama.cpp/requirements.txt`.** It pins `torch` to a CUDA 12.6 wheel, adds `tiktoken` (missing upstream, required for GLM), pins `transformers==5.3.0`, and narrows `numpy` to resolve an `opencv-python-headless` conflict. When bumping any of these, verify both constraints still hold. + +## Upstream path dependencies + +This repo hardcodes three paths inside `vendor/llama.cpp/`. Each build resets the submodule to the requested revision, so an upstream move breaks them at runtime with no build-time warning. After any `-version` / `-pullRequest` bump, a startup failure mentioning one of these should be treated as a relocation until proven otherwise: + +| Path | Used by | Failure signature | +| ---- | ------- | ----------------- | +| `gguf-py/gguf/scripts/gguf_dump.py` | `examples/server.ps1` metadata read | `Failed to extract model details` | +| `tools/server/bench/speed-bench/speed_bench.py` | `examples/speed-bench.ps1` | script not found at startup | +| `models/templates/google-gemma-4-31B-it.jinja` | gemma-4 preset entries | template read fails at model load | + +This is not hypothetical: upstream already relocated `gguf_dump.py` once, which is what CHANGELOG 1.24.0 records. + +## speed-bench + +- **`speed-bench.ps1` drives a router-mode server**, not a single model — it shells out to the vendored `vendor/llama.cpp/tools/server/bench/speed-bench/speed_bench.py` (wiped/refreshed each rebuild, so it tracks the built binary) and sweeps the `-models` preset ids in order, pre-warming each via the router-only `/models/load` endpoint and lazy-swapping through `--models-max 1`. Comparison anchors on the first id; models that fail to load are excluded, not fatal. Needs the `datasets` package (deliberately not in the main requirements) plus network access for the `nvidia/SPEED-Bench` dataset. The router-only `/v1/models` and `/models/load` endpoints mean it does not work against a plain single-model server. + +## Script argument parsing + +- **`server.ps1 -additionalArguments` splits on whitespace** and re-pairs tokens into key/value flags. Values that contain spaces will not survive this parser. + +## Build safety checks + +- **Rebuild aborts on running build-tree processes.** Before any destructive op, `rebuild_llama.cpp.ps1` checks `Get-Process` for any EXE under `vendor/llama.cpp/build/` and throws with the PID list. Catches the forgot-to-stop-`llama-server.exe` case. diff --git a/docs/model_tuning.md b/docs/model_tuning.md new file mode 100644 index 0000000..828cfd4 --- /dev/null +++ b/docs/model_tuning.md @@ -0,0 +1,558 @@ +# Model Tuning Reference + +Per-model-family rationale behind the entries in `presets/*.ini`, with the measured +numbers each decision rests on. Not auto-loaded into the agent context; read the +relevant section on demand. Cross-model rules are in `docs/presets.md`. + +## Qwen 3.6 and 3.8 + +- **Qwen-VL entries pin `image-min-tokens = 1024`.** `clip.cpp:1500` sets the per-image token limits + to `(8, 4096)`, so the default *minimum* is 8 tokens — 8192 px at `merge = 2` / `patch = 16` — and + `clip.cpp:1502-1506` warns on every load because upstream needs >= 1024 tokens (1024x1024 px) for + grounding (#16842). The key raises a floor only: images already above 1024 tokens are unchanged, + smaller ones get upscaled, which costs context and CLIP time on the CPU because the 16 GB tier + sets `no-mmproj-offload = true`; the 24 GB `Qwen3.8-27B` entry offloads CLIP to the GPU and + pays it there instead. Applies to the `qwen3vl_merger` entries (Qwen3.6, Qwen3.8 and + Ternary-Bonsai); gemma-4 uses a different projector and must not get this key. + +- **All Qwen 3.6, Qwen 3.8 and Bonsai entries pin `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`.** + Required, *not* redundant with `jinja = true` — `chat-template-file` *replaces* + the GGUF-embedded template entirely (`vendor/llama.cpp/common/arg.cpp:3142`, + `params.chat_template = read_file(value)`). The upstream embedded template has + documented issues with tool calls, role handling, `` block rendering, + agentic loops, and llama.cpp KV-prefix cache stability; the vendored template + fixes all of them (full list in `vendor/Qwen-Fixed-Chat-Templates/README.md`). + Since v19 the template is a single unified file, and since v22 it covers Qwen 3.5, + 3.6 *and* 3.8 (the old `qwen3.5/` and `qwen3.6/` subdirectories now live under + `archive/`). The template adds a `<|think_on|>` / `<|think_off|>` toggle, and + defaults `preserve_thinking` to `true` (past `` blocks are kept + chronologically for 100% KV prefix cache stability and agentic reasoning + continuity). To strip past `` blocks instead, set + `chat-template-kwargs = {"preserve_thinking":false}` — at the cost of a lower + KV cache hit rate. It also honours `preserve_reasoning`, so `--reasoning-preserve` + (`common/arg.cpp:3677-3689`) works as the CLI equivalent. Path is repo-relative, + so `llama-server` must be launched from the repo root — `read_file()` resolves + against the process CWD, not the INI file's directory. `Qwen3-Coder-Next` entries + deliberately keep their GGUF-embedded template; froggeric's README claims + compatibility only for Qwen 3.5 / 3.6 / 3.8 variants. + +- **Every Qwen 3.6 and Bonsai entry pins `reasoning-effort = medium`; the Qwen 3.8 entries pin + `xhigh`. Never leave the key unset.** The level is one injected paragraph at the top of the + system prompt — ~45 tokens of "Reasoning effort is set to xhigh..." — and `medium` is the single + level that injects nothing at all (`chat_template.jinja:53-59`). Qwen 3.6 has no trained notion of + the concept, so its entries pin `medium`; Qwen 3.8 *is* trained on it and Qwen's own template + defaults to `xhigh`, so those pin `xhigh`. The template's own default is a + `_default_reasoning_effort` variable at the top of the file (`chat_template.jinja:17`), currently + `medium`, and it has moved between template releases before. That is the whole reason to pin: + an unpinned entry silently changes reasoning level at a template bump, and a pinned one renders + byte-identically across bumps. `--reasoning-effort` writes only a template kwarg + (`common/arg.cpp:3650-3660`), which a request can still override + (`tools/server/server-common.cpp:1312-1319`), as can a `<|think_low|>` / `<|think_medium|>` / + `<|think_xhigh|>` tag typed inside a message (stripped before rendering). Unlike the + GGUF-embedded 3.8 template, the vendored one never raises on an unknown level: it maps `high`, + `max`, `ultracode` and `extreme` to `xhigh`, `minimal` to `low`, `none` and `off` to thinking + off, and anything else *down* to `medium` (`chat_template.jinja:21-26`). The `ultracode` and + `extreme` aliases exist for Claude Code, Cursor and Cline. + +- **`xhigh` is kept on `Qwen3.8-27B`, and `--reasoning-budget` — not a lower level — is the guard + rail for it.** Qwen publishes no per-level benchmarks; every number on the 27B card is at the + `xhigh` default, and the card warns that in multi-turn agentic tasks lower effort "can also lead + to insufficient analysis, more failures, and repeated retries". The entire mechanism is one + injected sentence: 123 rendered chars at `medium` against 332 at `xhigh` — no token, no sampling + change, no budget. The failure that moved froggeric's default to `medium` is a truncation + artifact rather than a quality result: with a finite `max_tokens` and no budget, `xhigh` ran + 26,000 tokens and returned `content_len = 0` because truncation landed inside ``, while + the same rig at `xhigh` with a 1500-token thinking budget returned 18,512 chars of working code, + more than `medium` produced. There is no rung between the two — `high` aliases to `xhigh`. + `--reasoning-budget N` (`common/arg.cpp:3662-3668`) is live on this entry and deliberately left + unset: the qwen3_coder handler supplies `` and `{"", ""}` + (`common/chat.cpp:1181-1184`), the server forwards them (`server-common.cpp:1360-1366`), and on + exhaustion the sampler masks every logit but the forced `` + (`common/reasoning-budget.cpp:119-131`, `:178-185`) so the model concludes instead of being cut + off, re-arming per thinking block (`:146-161`). Unlike `max_tokens` it counts only tokens inside + the block. Set it if an agentic client that sends its own `max_tokens` starts returning empty + content; a request can override it per call via `reasoning_budget_tokens`. + +- **`Qwen3.8-27B` pins the template too, even though its embedded one is already the newer file.** + Qwen 3.8 reuses arch `qwen35` and is otherwise byte-for-byte the same shape as + Qwen3.6-27B (65 blocks, 866 tensors, same `ssm.*`, same 248320-token tokenizer, + `eos = 248046`), but it embeds a *different, newer* 8952-byte template, not the + 7764-byte one shared by Qwen3.6-27B and both Bonsai variants. That newer file already + defaults `preserve_thinking` on and adds `reasoning_effort`, so it is a plausible + candidate for going unpinned. Three defects rule that out, all + reproducible by rendering the embedded template directly: tool calls whose `arguments` + arrive as a JSON *string* (what most OpenAI-compatible clients send) abort with + `Can only get item pairs from a mapping`; history carrying reasoning inside `content` + rather than `reasoning_content` renders a duplicate blank `\n\n` ahead + of the real block, because 3.8 dropped the in-content parser; and `reasoning_effort` + accepts only `xhigh` / `medium` / `low`, calling `raise_exception` on `high`, + `minimal` and `max` — three of the six levels `common/arg.cpp:3651` advertises. + The vendored template handles all three and its `xhigh` instruction text is + byte-identical to the official one, so `reasoning = on` with `reasoning-effort = xhigh` + reproduces the pre-pin prompt. Tool-call parsing is unaffected: the qwen3_coder XML handler is + selected purely on `` + `` block. The pin + additionally brings froggeric's agentic extras (two-tier tool-error escalation, + `<|think_on|>` / `<|think_off|>`, `developer` role, payload truncation). + +- **The vendored template (v22.4, pinned at `e649070`) corrects two v19 deviations from the + official Qwen prompt format, so moving to it changed every pinned entry, not just the new 3.8 + ones.** v19 serialized `` entries *unwrapped* + (`{"description": ..., "name": ..., "parameters": ...}`); the current template emits the wrapped + OpenAI form (`{"function": {...}, "type": "function"}`), which is what Qwen3.6-27B's own + 7764-byte template and Qwen 3.8's 8952-byte one both produce — v19 was the outlier. v19 also + rendered `\n` before assistant content where the official templates and llama.cpp's own + generation prompt use `\n\n` (`common/chat.cpp:1163`). Both are corrections, but they + change prompt bytes, so adopting the template invalidates existing KV prefix caches once; + `reasoning-effort = medium` suppresses only the steering paragraph and does not restore v19 + output. + +- **The template emits an empty `\n\n` before a historical tool call whose assistant + message carried no reasoning, and that is deliberate.** Qwen itself emits a think block before a + tool call when thinking is on, so injecting an empty one keeps rendered history token-aligned + with the model's own generation. It fires only when the client drops reasoning on the round trip; + supplying `reasoning_content`, `thinking`, `message.reasoning` (the vLLM and Responses API + spelling) or an inline `` block suppresses it. This is *not* the "empty think poisoning" + the template's README calls out — that was replacing *real* thoughts with empty blocks to save + tokens, which this template does not do. Verified by rendering the template against plain chat, + system-prompt, thinking-off, multi-turn-with-thinking, vision, and both tool-argument wire + formats; the tool-call case is the only one where an unset `reasoning_content` changes the + output. The same release line also brings the effort aliases above, reasoning de-duplication when + a client populates both `reasoning_content` and an in-content ``, complete serialization of + scalar and list tool arguments, and single-newline separation between consecutive `` + blocks for token parity on multi-tool turns. + +- **`Qwen3.8-27B` sets `temp = 1.0`, unlike the `0.6` used by the Qwen 3.6 entries.** + 1.0 is the official thinking-mode value on Qwen's card and is what the GGUF itself + embeds as `general.sampling.temp`, applied at `common/common.cpp:1264` unless the + preset overrides it. The rest of the sampler block (`top-p 0.95`, `top-k 20`, + `min-p 0.0`, `presence-penalty 0`) is unchanged from the 3.6 entries. Qwen 3.8's + non-thinking mode wants a different set (`temp 0.7`, `top-p 0.8`, + `presence-penalty 1.5`); the preset does not cover it because `reasoning = on`. + +- **Qwen 3.8's MTP head is multi-step trained, so `spec-draft-n-max = 3` is a measured + peak rather than an inherited default.** A day-0 `n_max` sweep of 2/3/4/6 on this model + put the maximum at 3 — acceptance falls monotonically with depth, but through 3 the + extra tokens per iteration win. This overturns the Qwen 3.6 rule of thumb that 2 was + optimal. 3 also happens to be the upstream default (`common/common.h:325`). Note the + cost: `draft-mtp` sets `n_rs_seq = spec-draft-n-max` (`common/common.cpp:1699`), which + multiplies the recurrent-state buffer by `1 + n_max` — ~150 MiB becomes ~600 MiB at 3. + Both `Qwen3.8-27B` and `Qwen3.6-27B` carry `blk.64` (the MTP head) at `Q4_0` in the + local IQ4_XS files; a 4-bit MTP head is reported to collapse acceptance to 0% on this + model family, so check the server's acceptance rate before trusting the speedup. Two + remedies exist: a re-quant keeping `blk.64` at `Q5_K` or above, or — since 2026-08-14 — + pointing `spec-draft-model` at `ggml-org/Qwen3.8-27B-GGUF`'s standalone + `mtp-Qwen3.8-27B-Q8_0.gguf`. Neither is taken. The sidecar is 3.16 GB against the + ~2.65 GiB of headroom measured below, the `Q4_0` sidecar is the same precision as the + embedded head, and `mparams.load_mtp` is set from the *type list* rather than from the + presence of a draft path (`common/common.cpp:1689`, `src/models/qwen35.cpp:42`), so the + target keeps loading its own `blk.64` and an external sidecar double-pays. + +- **`Qwen3.8-27B` is the only 24 GB Qwen entry on a `Q8_0` projector instead of `BF16`.** + 600 MiB rather than 888 MiB of VRAM, and that saving is what keeps `mmproj-offload = true` + affordable at `ctx-size = 262144`: the entry lands at ~19.95 GiB of ~22.6 GiB usable, just + under the Qwen3.6-27B entry's ~20.23 GiB, which leaves room for the CLIP compute buffer. + Spending the saving elsewhere is what breaks it — raising the KV cache to `q5_0` K / `q4_1` V + costs ~0.80 GiB at this context and pushes the total above the 3.6 entry, into the + silent-OOM window described in `docs/presets.md` -> *mmproj-offload*. Quality is not the tradeoff: Qwen ships this family's + projector as FP16 *and* `Q8_0` officially, and only 83 of the file's 110 weight tensors are + actually 8-bit — every `ffn_down` stays `F16`. + +- **Quantize Qwen3.8 GGUFs from `Qwen/Qwen3.8-27B`, never from `Qwen/Qwen3.8-27B-FP8`.** BF16 is + this model's native precision and the FP8 repo is a derived, post-training artifact (HF model + tree: base model `Qwen3.8-27B`, "Quantized"), so it is already lossy — its card claims only + "nearly identical" metrics. Quantizing from it would fit the quantizer and the imatrix to + degraded weights. This is the opposite of DeepSeek-V3/V4, which were *trained* in FP8, making + their FP8 checkpoint the original and its dequant to BF16 exact; `convert_hf_to_gguf.py:156` + (`--fp8-as-q8`) exists for that case, not this one. The mmproj is the one exception where the + source does not matter: the FP8 repo leaves the whole vision tower unquantized (0 of 333 + `model.visual.*` tensors carry `weight_scale_inv`), so it is byte-identical either way. The + MTP head is not — `mtp.layers.0`'s attention and MLP projections are FP8 there, so any + re-quant raising `blk.64` above 4-bit must also come from the BF16 repo. + +## Qwen3.8-Flash-Next + +Arch `qwen4exp` (upstream #27742, merged at `b10660`), a separate architecture from the `qwen35` +family above and tuned on different grounds. 48 blocks: 12 full-attention layers carrying QSA +block-sparse attention over an indexer cache, 36 gated-delta-net layers, 512 experts with 10 used, +`context_length = 262144`, `qwen3vl_merger` projector. + +- **`load-mode = mmap` and `no-host = true` are one mechanism, not two independent choices.** The + 26.822 GiB `per_layer_token_embd` n-gram hash table (`Q4_0`, 160 x 320,001,536) is created with + `TENSOR_READ_LAZY` (`src/models/qwen4exp.cpp:139-140`), and the loader gates that flag on + `use_mmap` (`src/llama-model-loader.cpp:1290`), which only `mmap` / `mmap+mlock` / `auto` set + (`:559`). The `auto` threshold is 4 GiB (`:1292`), so the table qualifies without pinning + `tensor-read-lazy`. A token gathers `ple_n_heads = (ngram_size - 1) * heads_per_ngram` = 16 rows + (`src/models/qwen4exp.cpp:64`, gather at `:1106-1112`), so a session touches a vanishing fraction + of the 320 M rows and mmap — which aliases the file rather than copying it + (`src/llama-model-loader.cpp:1578-1602`) — keeps the resident working set in the hundreds of MiB. + `dio` reads and holds all 26.822 GiB for nothing; this is the one entry in the tier that must not + use it. `no-host` is what keeps that path reachable: without it `make_cpu_buft_list` prepends + `CUDA_Host` to the CPU list (`src/llama-model.cpp:1047-1049`), the chosen buft then fails the + `is_default_buft` test at `:1715`, the mmap-aliasing branch at `:1718` is skipped entirely, and + every CPU-resident tensor — tens of GiB of experts plus the 26.822 GiB table — goes through + `cudaMallocHost`, which is the misleading CUDA OOM documented for DeepSeek below. `mmap+mlock` is + the other wrong answer: it forces the whole table resident + (`src/llama-model-loader.cpp:1595-1598`). + +- **The table can never be offloaded, so `-ngl` is not the lever and the entire budget question is + expert layers.** `LLM_TENSOR_PER_LAYER_TOKEN_EMBD` is classified `LLM_TENSOR_LAYER_INPUT` + (`src/llama-arch.cpp:887`) and `src/llama-model.cpp:1482-1483` pins every input tensor to + `cpu_buft_list` regardless of `-ngl` ("there is very little benefit to offloading the input + layer"); only an explicit `-ot` could move it. Composition of the local IQ4_XS (90.635 GiB, 1224 + tensors): 26.822 GiB PLE table plus 0.333 GiB `token_embd` on the CPU by construction, 60.938 GiB + of routed experts (1.270 GiB per layer x 48) for `fit` to place, and 2.542 GiB of everything else + on the GPU. The GPU-side floor is therefore small, and how many of the 48 expert layers survive + beside the KV cache is the only thing that moves throughput. + +- **`fit = on` with an explicit `ctx-size`, and fit is blind in two ways here.** Fit reduces context + only when the user left it unset (`common/fit.cpp:197` `n_ctx_auto = n_ctx == 0`, used at `:392`), + so `ctx-size = 262144` is honoured and step 3 spends the remainder on expert fractions — its + overflow pattern `blk\.N\.ffn_(up|down|gate_up|gate)_(ch|)exps` (`common/fit.cpp:522`) matches + exactly what this arch names them (`src/models/qwen4exp.cpp:202-203`). Blind spot one: fit assumes + host memory is unlimited (`common/fit.h:24`) and never consults the CPU slot once a GPU is present + (`common/fit.cpp:331-347`). Blind spot two: it measures with `load_mode = LLAMA_LOAD_MODE_NONE` + (`common/fit.cpp:57`), so lazy read is off during measurement and its host figure counts the full + PLE table — a number the real run never produces. Neither matters on a 191 GiB box, but they are + why the split cannot be sanity-checked from fit's own host accounting. Setting `n-gpu-layers` to + anything but `-1`, or any `-ot` / `--cpu-moe` / `--n-cpu-moe`, throws inside fit + (`common/fit.cpp:462-464`, `:483-485`) and is downgraded to a warning at `:893-895`: fit silently + does nothing and the model no longer fits at all. + +- **`cache-type-k` also types the QSA indexer cache, which is why K stays at `q8_0`.** + `src/llama-model.cpp:2506-2507` hands `params.type_k` / `type_v` to `llama_memory_hybrid_idx`, + which forwards them unchanged to the indexer cache (`src/llama-memory-hybrid-idx.cpp:56`). + Indexer K is what `ggml_top_k` ranks blocks on (`src/models/qwen4exp.cpp:599-601`), so cheapening + it degrades *which* tokens are attended, not just their values. Per-token cost at + `kv-unified = true`: 12 attention layers x 1024 elements (`n_head_kv = 2` x `head = 256`, K and V) + plus 12 indexer layers x 384 elements = 17,952 B at `q8_0`, i.e. 4,488 MiB at 262144. Two thirds + of the indexer share is dead — `src/llama-memory-hybrid-idx.cpp:50-51` sets `n_embd_head_k_full` + but not `n_embd_head_v_full`, so `is_mla()` is false, `src/llama-kv-cache.cpp:232-235` allocates a + 256-wide V, and the graph only ever calls `cpy_k` / `get_k` (`src/models/qwen4exp.cpp:530`, + `:533`). Dropping `cache-type-v` to `q4_0` would recover 1,152 MiB at this context with the + quality cost paid only by the 12 real attention layers; worth trying, not taken. Unlike + `deepseek4` the two types may legally differ — the equality guard at + `src/llama-context.cpp:3592-3595` fires only for `is_mla()` or `LLM_ARCH_DEEPSEEK4`. + +- **Recurrent state is 112.219 MiB per sequence and independent of `ctx-size`, so `parallel` is the + cheap knob and context the expensive one.** 36 gated-delta-net layers at + `n_embd_r = 3 x 10240` and `n_embd_s = 128 x 6144` elements (`src/llama-hparams.cpp:204`, `:232`), + both hardcoded `GGML_TYPE_F32` (`src/llama-model.cpp:2513-2514`) so no cache type shrinks them, + and one row per sequence because `qwen4exp` is absent from `llm_arch_supports_rs_rollback` + (`src/llama-arch.cpp:1099-1113`) and `n_rs_seq` is clamped to 0 at + `src/llama-context.cpp:105-108`. The entry runs `parallel = 4` on the strength of that: with + `kv-unified = true` `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`), so four slots *share* + the 262144-cell pool rather than each being given one, the KV cost is unchanged, and the only + VRAM the extra slots add is three more recurrent rows — 336.7 MiB. A single long conversation can + still occupy the whole pool. + +- **A context checkpoint here is the entire recurrent state, ~112 MiB, and checkpoints are per + slot — which is why `ctx-checkpoints` is 8 and not 32.** Checkpoints are written with + `LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY`, and that flag skips both the attention cache + (`src/llama-memory-hybrid.cpp:191-192`) and the indexer cache + (`src/llama-memory-hybrid-idx.cpp:204-206`), leaving only + `llama_memory_recurrent::state_write` (`src/llama-memory-hybrid.cpp:194`). So the blob is the full + 112.219 MiB rather than the 14.5 MiB a DSV4 checkpoint costs, and because + `slot.prompt.checkpoints` is per slot (`tools/server/server-context.cpp:2283`) the host budget is + `parallel x ctx-checkpoints x 112 MiB`. At `parallel = 4`, 32 checkpoints would reserve up to + 14.3 GiB of host RAM; 8 holds it at the ~3.6 GiB that `parallel = 1` with 32 would have cost, and + at the shipped `parallel = 2` it is ~1.8 GiB. Raising `parallel` again means lowering this in step. + +- **Measured on a 24463 MiB card, `q8_0` K + V, CLIP on CPU.** At `ctx-size = 262144` / + `parallel = 1`: 20174 MiB used, 3964 MiB free, 19.87 t/s tg at short context. At the shipped + `ctx-size = 524288` / `parallel = 2`: 19950 MiB used, 4513 MiB free, 14.90 t/s on a 400-token + prose completion. The two throughput figures are *not* a controlled comparison — different + prompts, and `ngram-mod` acceptance dominates on predictable output (the same config returns + 22.51 t/s counting to 60). "Used" barely moves between configs because `fit` always fills to the + `fit-target` margin; what changes is the composition. Throughput is expert-traffic bound, not + attention bound: every token reads 10 of 512 experts across all 48 layers, ~26.1 MiB per layer at + IQ4_XS, so the ~1.25 GiB per token that is not resident on the GPU is what sets the rate. That is + the currency `ctx-size` is spent in — 1,300 MiB of KV is one expert layer is roughly 2 % of tg. + +- **`ctx-size = 524288` with `parallel = 2` buys two concurrent full-length conversations, and + costs about half the GPU-resident expert layers to do it.** `llama-fit-params` gives the fixed + cost directly, in MiB, as a function of `n_ctx_seq` (`context` is KV plus the recurrent rows, + `compute` is the graph buffer): + + | `n_ctx_seq` | context | compute | fixed total | + |------------:|--------:|--------:|------------:| + | 65,536 | 1,234 | 573 | 1,807 | + | 262,144 | 4,600 | 1,821 | 6,421 | + | 524,288 | 9,088 | 3,497 | 12,585 | + | 1,048,576 | 18,064 | 6,829 | 24,893 | + + Add 112 MiB per sequence beyond the first for the extra recurrent row. So the shipped config's + fixed cost is 12,697 MiB against 6,758 MiB for 262144 / `parallel = 4`; the ~5,900 MiB difference + comes straight out of expert layers, taking them from ~8.3 of 48 to ~4.4. The benefit is real but + narrow: under `kv-unified` `n_ctx_seq = n_ctx` (`src/llama-context.cpp:290-291`) while the server + still caps each *slot* at `n_ctx_train = 262144`, so a single conversation is capped at 262144 + either way and only a *second* concurrent long conversation can reach into the extra cells. On a + single-user workload the larger pool is paid for and unused. + +- **A 1,048,576-cell pool does not fit, and the ceiling is arithmetic rather than a tuning + question.** Going from 262144 to 1048576 takes the KV cache from 4,488 to 17,952 MiB at `q8_0`, + `+13,464 MiB`, against 3,964 MiB measured free plus at most ~9 GiB recoverable by moving every + remaining expert layer to the CPU — and the QSA bias tensors are sized `[n_kv, n_tokens]` + (`src/llama-memory-hybrid-idx.h:132-138`), so the compute buffer grows by the same factor of four + on top. Even if it squeezed in it would be a regression, because zero expert layers on the GPU is + strictly slower than the current split. `parallel` is not a way around it: under `kv-unified` the + slots share one pool, so wanting four slots that can each reach 262144 *is* asking for 1,048,576 + cells, at exactly the cost tabulated above. The only + route to a 1 M pool on 24 GB is `q4_0` K and V, which halves it to 9,504 MiB, and `cache-type-k` + is precisely the value that should not drop because it types the indexer. Note that a large pool + needs no `override-kv`: `n_ctx_train` is 262144, so the server caps each slot there + (`tools/server/server-context.cpp:1209-1214`, applied at `:1274`) and the `- capping` line is + expected rather than a misconfiguration. + +- **No MTP head, and context shift and cache-reuse are structurally impossible — which is what makes + `ctx-checkpoints` load-bearing.** `conversion/qwen4exp.py:28-30` drops the MTP block ("a separate + draft head; vLLM drops it too"), so the GGUF carries no `nextn` tensors and `spec-type = + draft-mtp` fails at `src/llama-context.cpp:3637-3642`; `ngram-mod` is the only speculative type + available. `get_can_shift()` is false because IMRoPE gives `n_pos_per_embd() == 4` + (`src/llama-kv-cache.cpp:1194-1196`, rope type at `src/llama-model.cpp:2951-2955`), so the server + force-disables context shift *and* cache-reuse with two warnings at + `tools/server/server-context.cpp:1185-1195` — both are expected on startup, not a + misconfiguration. Speculative rollback then goes through checkpoints + (`tools/server/server-context.cpp:1224-1226`), so a non-zero `ctx-checkpoints` is required for + `ngram-mod` to be useful rather than being an optimisation. `swa-full` is inert: `swa_type` is + `NONE`, which is why the model takes the `hybrid_idx` path at `src/llama-model.cpp:2502` at all, + and the server clears the flag at `:1197-1202`. + +- **The template pin is the same file as `Qwen3.8-27B`'s, byte for byte.** The embedded template is + 8952 bytes with sha256 `c3cf9e34abf4f9e3...` — identical to Qwen3.8-27B's — so the three defects + documented above (tool-call `arguments` arriving as a JSON string, a duplicate blank + `\n\n` from history, `raise_exception` on `high` / `minimal` / `max`) apply + unchanged, and so does the fix. `reasoning-effort = xhigh` for the same reason; `--reasoning-budget` + remains the guard rail rather than a lower level. Pinning costs no vision: the vendored template + renders `<|vision_start|><|image_pad|><|vision_end|>` + (`vendor/Qwen-Fixed-Chat-Templates/chat_template.jinja:67-88`). The projector is + `qwen3vl_merger`, so `image-min-tokens = 1024` applies as it does to the other Qwen-VL entries. + The sampler block restates the GGUF's own `general.sampling.*` (`temp 1.0`, `top-p 0.95`, + `top-k 20`), matching Qwen3.8-27B. + +- **`no-mmproj-offload = true`, because CLIP is exactly what `fit` cannot see.** Fit measures the + language model alone and commits the expert split before `mtmd` loads the projector, so a + `mmproj-offload = true` here lands in the silent-OOM window described in `docs/presets.md` -> + *mmproj-offload* with no margin left to absorb it — the 588 MiB of `Q8_0` weights plus ~310 MiB of + CLIP compute would have to be reserved by hand through `fit-target`. `fit-target = 3072` is the + same WDDM-plus-untracked-CUDA-scratch margin as the DeepSeek entry on the same card, for the same + reasons. `cache-ram = 32768` rather than the 51200 used by the small entries: a full-context + prompt state is ~4.6 GiB here (4,488 MiB KV plus the recurrent rows), and the host is already + backing most of a 90.635 GiB file through the page cache plus a PLE working set that only grows. + +## gemma-4 + +- **All gemma-4 entries pin `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja`.** + This is Google's fixed official template as aligned by upstream (#21704) — the exact + file upstream's `tests/test-chat.cpp` locks against the native gemma4 chat handler + (`vendor/llama.cpp/common/chat.cpp:1216`), so parser and template always come from the + same submodule commit (each rebuild resets the submodule to master, mirroring the built + binary). GGUF-embedded templates from conversions predating Google's template fixes + lack the `{#- OpenAI Chat Completions:` marker; llama.cpp then logs "detected an + outdated gemma4 chat template" and rewrites messages via C++ compatibility workarounds + (`common/chat.cpp:2250-2258`) — the pin avoids that path. One file covers the whole + series (12B / 26B-A4B / 31B, incl. `<|image|>`/`<|audio|>` placeholders), and + `reasoning = on` maps to the template's `enable_thinking` kwarg + (`common/arg.cpp:3167-3175`), so no `chat-template-kwargs` are needed. Unlike the Qwen + template, past `<|channel>thought` blocks are *stripped* from history by design — + Gemma 4 is trained that way — so cross-turn KV-prefix invalidation is inherent + (`ctx-checkpoints` mitigates); do not add a preserve-thinking hack. If startup fails + reading the template after a rebuild, check whether upstream moved + `models/templates/` — see `docs/build_system.md` -> *Upstream path dependencies*. + +## Bonsai and DSpark + +- **Both Bonsai entries use the same `chat-template-file` pin as the Qwen 3.6 entries.** + `Ternary-Bonsai-27B` and `Bonsai-27B` ship from separate HF repos but are both Qwen3.6-27B + derivatives: arch `qwen35`, and their tokenizers are byte-identical to stock Qwen3.6-27B + (248320 tokens, same merges, `eos = 248046`) right down to the same 7764-byte embedded template — + which is exactly the upstream template the pin exists to replace. `general.sampling.temp = 1.0` is + embedded in both GGUFs and applied at `common/common.cpp:1264`, so `temp` has to be pinned in the + preset or generation runs at 1.0. The presets use `0.6` to match the sibling Qwen 3.6 entries; + Prism's own card benchmarks at `0.7`. Unlike the DSpark sidecar below, both weight files are + mainline-packed (`Q2_0` at `QK2_0 64`, `Q1_0` at `QK1_0 128`) and load without a tensor-offset + mismatch. + +- **The DSpark drafter shipped beside Ternary Bonsai 27B cannot be enabled on mainline.** + `Ternary-Bonsai-27B-dspark-Q4_1.gguf` has its `token_embd.weight` in `Q2_0` at Prism's + group-128 packing while mainline is group-64 (`QK2_0 64`, `ggml/src/ggml-common.h`), so + `gguf_init_from_reader` rejects the file on a tensor-offset mismatch before any architecture + dispatch. Repacking would not help: `general.architecture = 'dspark'` is unregistered + (`src/llama-arch.cpp:136` has only `dflash`), and mainline's DSpark is DeepSeek-V4 + DFlash + Markov (`src/models/dflash.cpp`, tensors `markov_w1`/`markov_w2`/`conf_proj`, + requiring MLA and sqrtsoftplus MoE scoring), not Prism's 6-layer Qwen3.6-shaped drafter + (`dspark.fc`, `dspark.log_snr_fc*`, `dspark.markov_head_*`). Upstream confirmed on #25707 + that it stays fork-only. The GGUF carries no MTP tensors either, so `draft-mtp` is out and + the entry uses `ngram-mod`. Only the group-64 pack is mainline-loadable — `Q2_0.gguf` and + `PQ2_0.gguf` in the same HF repo are group-128 fork packs. + +## DeepSeek-V4-Flash + +- **DeepSeek-V4-Flash-0731 must set `cache-type-k` *and* `cache-type-v` to the same value.** + Arch is `deepseek4` (the HF card's "dflash / 20B" is the DSpark sidecar's metadata, not the + model). `llama-context.cpp:3560-3563` compares the two values and refuses to create the + context — `does not support different K (%s) and V (%s) cache types` — for + `LLM_ARCH_DEEPSEEK4` specifically, because `hparams.is_mla()` is *false* for this arch and the + guard needs the explicit disjunct. So `cache-type-v = q8_0` is load-bearing even though V is + never allocated: DSV4 is K-only everywhere (`dsv4_make_k_only()` at + `llama-kv-cache-dsv4.cpp:831-835` forces `is_mla` true on hparams copies, so + `has_v = !is_mla` at `llama-kv-cache.cpp:229` is false). Copying the Qwen dual-GPU pair + `q5_0` K / `q4_1` V here is startup-fatal, not merely wasteful. `q8_0` also clears + `n_embd_head_k() % 64 == 0` so quantized K gets the Hadamard rotation + (`llama-kv-cache.cpp:319-323`); the lightning-indexer cache is rotated unconditionally for + this arch (`:325-329`). `kv-unified` is silently discarded (`GGML_UNUSED(unified)`, + `dsv4.cpp:1189-1192`), and `cache-type-*-draft` is dead without a draft model. + +- **The DSV4 KV cache is tiny, so context is cheap and quantizing it buys little.** 43 layers + split 2 raw / 21 CSA (ratio 4) / 20 HCA (ratio 128) via `attention.compress_ratios`, all + K-only at `n_embd_k_gqa = 512`; the raw tier is SWA-windowed to + `PAD(min(n_ctx, 128 + n_ubatch), 256)` = 768 cells regardless of `ctx-size`. At 262144 that + is 942 MiB at `q8_0` (1764 MiB at f16) plus a fixed 11.64 MiB of F32 compressor state that no + cache type shrinks. Never set `swa-full`: it collapses the window formula to `n_ctx` + (`llama-kv-cache-iswa.cpp:76-81`), turning a 17 MiB raw cache into ~11 GiB. The server warns + `swa_full is not supported` only *after* the cache is built, so the flag still takes effect. + +- **Leave `fit = on` and never add `n-cpu-moe`/`-ot` to the DeepSeek entry.** Measured UD-Q8_K_XL + composition: 137.06 GiB routed experts (MXFP4, 90.9%), 2.02 GiB shared experts, 11.67 GiB + non-expert — so non-expert + shared is only 13.69 GiB and fits a 24 GB card alongside the KV + with room for a full expert layer or two (3.19 GiB each). `fit` finds that split at sub-layer + granularity (`fit.cpp:399-441`, `:719-769`); forcing `-ncmoe 43` would push all experts to CPU + and strand ~9 GiB of VRAM. Any user `-ot`/`--cpu-moe`/`--n-cpu-moe` aborts fit outright + (`fit.cpp:395-397`), as does setting `n-gpu-layers` to anything but `-1` (`fit.cpp:374-376`) — + which is why the entry keeps `-1` explicitly. Note `--cpu-moe`'s pattern matches only + `_exps`/`_chexps`, so shared experts would stay on GPU either way. + +- **`no-host = true` is mandatory on the DeepSeek entry, and this is the trap that actually stops + it loading.** Unless `no_host` is set, `make_cpu_buft_list()` prepends + `ggml_backend_dev_host_buffer_type()` to the CPU buffer list, so *every* CPU-resident tensor is + allocated in a `CUDA_Host` (page-locked) buffer (`src/llama-model.cpp:896-917`, wired from + `params.no_host` at `common/common.cpp:1611` and `include/llama.h:338`). For this model the + loader then reports one `CUDA_Host model buffer size = 137046.96 MiB` — a 133.8 GiB + `cudaMallocHost` on a 192 GB box. The reservation *succeeds*, so `ggml_cuda_host_malloc`'s + clean-failure fallback to an ordinary CPU buffer never fires; the failure happens later while + the pages are committed during the read and surfaces as `CUDA error: out of memory` inside + `cudaEventSynchronize` at `src/llama-model-loader.cpp:1591`. That makes a host-memory problem + look like a VRAM problem — raising `fit-target` does not help it, and neither does changing + `load-mode`. With `no-host = true` the same config loads in ~144 s at 18650 MiB VRAM and + ~124 GiB of ordinary host RAM. `GGML_CUDA_NO_PINNED=1` is the env-var equivalent. This applies + to any entry that pushes tens of GiB of experts to CPU, not only DeepSeek. The GPU upload + staging buffers are unaffected — the loader asks for those buffer types directly + (`src/llama-model-loader.cpp:1467`) rather than through `cpu_buft_list` — but expert weights + that `op-offload` ships to the GPU for large-batch matmuls now come from pageable memory, which + may cost some prompt-processing throughput. There is no way to keep that and still load. + +- **`fit-target = 3072` on the DeepSeek entry is a WDDM safety margin, not the fix for the load + failure** (that is `no-host` above). `fit` measures rather than guesses — it performs a + `no_alloc` model load plus a real graph reservation (`fit.cpp:56-75`), so its KV figure is + byte-exact (942 MiB at 262144/`q8_0`) and its compute figure is a genuine `ggml_gallocr` + measurement. What it cannot see is the CUDA VMM scratch pool (32 GiB of VA reserved, physical + pages committed on demand, `ggml/src/ggml-cuda/ggml-cuda.cu:536-656`), the lazy cuBLAS + workspace, and CUDA graph instances; none are reported to `memory_breakdown()`. It also takes a + single `cudaMemGetInfo` snapshot at t=0 (`fit.cpp:194`) and carries no WDDM or framebuffer + allowance anywhere. At the default 1024 MiB margin fit keeps blk.0 and blk.1 routed experts on + the GPU (6.375 GiB — every one of the 43 layers carries 3.188 GiB of routed experts, there are + no dense layers) and leaves only 1368 of 23139 usable MiB for those untracked consumers. + `3072` leaves 3497 MiB and costs one extra expert layer on CPU (~2.3% more expert traffic). + Do not raise it to 6144: that collapses `-ngl` to 38 and starts stranding whole layers. + `--fit-target` writes only `params.fit_params_target` and never `mparams`, so unlike + `-ngl`/`-ncmoe`/`-ot` it cannot trip the aborts at `fit.cpp:374-397`. + +- **`cache-ram` is 16384 on the DeepSeek entry, not the 51200 used elsewhere.** `fit` reports + 133.8 GiB of `Host model` weights for this entry (measured at `fit-target = 3072`), so on a + 192 GB box a 50 GiB prompt cache overcommits and pages. A full-context prompt + state at 262144/`q8_0` is 931 MiB (`server-task.cpp:1671-1683` — an entry larger than the + whole limit is silently skipped), so 16 GiB still holds ~17 of them. Context checkpoints are + separate and cheap: 14.5 MiB each and independent of `ctx-size`, because a DSV4 checkpoint + stores only the 128-position SWA window plus the fixed compressor state. + +- **The DeepSeek-V4-Flash entry deliberately does *not* pin `chat-template-file`.** This is the + one exception to the convention above. The GGUF ships Unsloth's fixed template, and both it + and upstream's bundled `models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja` (#26398) + satisfy the detection heuristic at `common/chat.cpp:3170-3179` (`dsml_token` + `DSML` + + `tool_calls`), so both route to the native PEG parser + (`common_chat_params_init_deepseek_v3_2`, `chat.cpp:2097`) and classify as V4 via the + `function_calls`-absent test at `:2105`. Unsloth's additionally restores `reasoning_content` + on tool calls, which the official template drops. Unlike gemma-4 there is no outdated-template + rewrite path for `deepseek4` — detection is all-or-nothing, and a miss degrades to the generic + autoparser rather than being repaired. The entry carries the effort as + `chat-template-kwargs = {"reasoning_effort":"high"}` (or `"max"`); `reasoning-effort = high` is + equivalent and simpler — `common/arg.cpp:3650` writes the same template kwarg, and the level + `xhigh` is listed in its help text. Only the *request* field special-cases the literal `"none"` + (`server-common.cpp:1296-1304`). Left unset the template defaults it to `none` and emits no + effort block at all, and `reasoning = off` voids it entirely. Do not set `reasoning-format`: the compiled default is already `deepseek` + (`common.h:631`, despite the help text saying `auto`), and `none` leaks `` into + `content`. + +- **Context shift and cache-reuse are permanently unavailable on `deepseek4`.** + `llama_kv_cache_dsv4::get_can_shift()` returns false (`dsv4.cpp:1394-1398`), so the server + force-disables both with a warning (`server-context.cpp:1268-1278`); slots then stop cleanly + at `STOP_TYPE_LIMIT` instead of shifting. `seq_rm` also refuses partial removal when + `n_rs_seq == 0` (`dsv4.cpp:1427-1429`), which `ngram-mod` does not set, so rollback goes + through checkpoints — correct, but each rejected draft costs a ~14.5 MiB state restore and the + net throughput effect is uncharacterised. The `dspark` sidecar in the same HF repo is a + genuine mainline `dflash` drafter (unlike the Bonsai one above), but is unusable here: its + README requires `--fit off` plus full offload of target *and* drafter (11 GiB drafter + + 13.69 GiB non-expert exceeds 24 GB), `--spec-draft-n-max` is clamped to 5, multi-GPU needs a + rebuild with `GGML_SCHED_MAX_SPLIT_INPUTS=48`, and it carries an open decode-time CUDA abort + after ~2500 tokens (#26554). The regression that broke spec decoding on this arch (#26576, + a 2D `wo_a` in `dflash.cpp` after #26531) is fixed by #26577 at `b10269`. + +## Muse Glimmer + +- **Muse Glimmer has no architectural positional ceiling, so never add RoPE scaling.** Meta's + `config.json` sets `layer_rope_theta = 0` on all 13 `full_attention` layers (NoPE) and + `sliding_window = 2048` on the other 39; the GGUF's `sliding_window_pattern` confirms the + `[true, true, true, false]` x13 split. RoPE therefore never sees a relative position above 2048 + at any context length — 131072 is a training length, not a limit, which is why Meta writes + "131,072+". Because the GGUF carries no `rope.scaling.*` keys the type defaults to `linear` with + `freq_scale 1` (`src/llama-model.cpp:1187-1189`), so a user `--rope-scale` is *applied*, not + ignored, and would compress local resolution on the SWA layers for no benefit. The + `--rope-scaling yarn` recipes circulating on the HF model page are wrong; only their + `--override-kv` half is load-bearing. + +- **Never set `swa-full` or `context-shift` on a Muse Glimmer entry.** `swa-full` collapses the + 4608-cell SWA cache to the full `n_ctx` by the mechanism documented for DeepSeek above — a ~114x + blowup at `ctx-size = 524288`. `context-shift` is the more dangerous one, and it is the exact + inverse of the `deepseek4` case: `llama_hparams::has_rope()` knows only about `router_layer` + (`src/llama-hparams.cpp:287-295`), so a K-shift rotates 128 dims on all 52 layers including the + 13 NoPE ones, while `get_can_shift()` returns true. Silent corruption rather than a refusal. It + is dormant only because `ctx_shift` defaults false (`common/common.h:561`) and loading an mmproj + force-disables it (`server-context.cpp:1163-1171`); dropping the mmproj would make it reachable. + +- **`spec-draft-n-max = 15` is `block_size - 1`, and the absent `chat-template-file` and + `reasoning` keys are both deliberate.** The drafter carries `dflash.block_size = 16` and spends + block position 0 on the committed anchor, so 15 is the maximum legal draft + (`common/speculative.cpp:970-978`, no clamp and no warning); the upstream default is 3 + (`common/common.h:325`), so leaving it unset discards 80% of a block that is decoded in one + forward pass regardless. Judge it by `mean len` in the slot timings + (`1 + n_accepted / n_verif_steps`, `server-context.cpp:620`), never by `draft acceptance` — that + ratio is a percentage of *drafted* tokens and is meaningless for block diffusion, where + unaccepted drafts cost nothing. Measured ~3.4 tokens per target pass at short context, ~3.0 at + 4.3k. The template is deliberately not pinned: `common/chat.cpp:3494` selects the native Muse + Glimmer handler on `` + `<|eom|>` in the template source, and the bundled + `models/templates/muse-glimmer.jinja` is a test fixture added alongside a `chat.cpp` parser fix + (#26879, locked by `tests/test-chat.cpp:5951`) that mirrors the GGUF-embedded template — unlike + gemma-4, where the pin exists to replace genuinely outdated conversions. + `reasoning = on` is a no-op — neither the handler nor the GGUF's embedded template reads + `enable_thinking`; reasoning here is the structural ` to=self<|message|>...<|eom|>` channel, + gated on `reasoning_format` (`chat.cpp:3141`). The template already defaults to + `Reasoning strength: high` (`models/templates/muse-glimmer.jinja:84`), which matches Meta's + recommendation, and `--reasoning-effort` reaches it only through the + `reasoning_effort` -> `reasoning_strength` alias at `common/jinja/caps.cpp:29-33`. + `--reasoning-budget` is inert: the handler sets no `thinking_end_tags` + (`server-common.cpp:1360`). + +- **Compute buffers dominate this entry's headroom and cannot be derived from KV arithmetic.** + Measured on a 24463 MiB card at `ctx-size = 524288` / `parallel = 2` / `q5_0` K + `q4_1` V: + 13488.92 MiB target weights, 1371.40 draft, 1956.60 CLIP, 2271.12 KV (2184.00 non-SWA + + 57.59 SWA + 29.53 draft) and **2655.59 MiB of compute buffers** (1135.67 target + 803.03 + spec-context + 407.62 draft + 309.27 CLIP) — 21743.63 MiB total, leaving ~1.0 GiB free. The + compute term is larger than the whole KV cache and grew 1070.67 -> 1135.67 MiB once a real + request arrived, so budget it explicitly instead of sizing `ctx-size` off KV alone. Note the + SWA cache scales with `parallel`, not `ctx-size` + (`PAD(min(n_ctx, n_swa * n_seq_max + n_ubatch), 256)` = 4608 cells), so raising `parallel` + is cheap while raising `ctx-size` is not. `mmproj-offload = true` survives this margin (CLIP + warmup reserves 309.27 MiB and real images decode), but it is the first thing to disable if the + margin shrinks — see `docs/presets.md` -> *mmproj-offload*. Context checkpoints are host-side, ~38.8 MiB + each at `max = 32`. diff --git a/docs/presets.md b/docs/presets.md new file mode 100644 index 0000000..66e932d --- /dev/null +++ b/docs/presets.md @@ -0,0 +1,93 @@ +# Presets Reference + +Cross-model rules for editing `presets/*.ini`. Not auto-loaded into the agent +context; read it on demand. `presets/README.md` is the user-facing quick-start; +this file is for editing. Per-model rationale lives in `docs/model_tuning.md`. + +## Editing presets + +VRAM-tier presets: `presets/models_16GB_VRAM.ini`, `presets/models_24GB_VRAM.ini`, +`presets/models_16GB_8GB_VRAM.ini` (dual-GPU). + +## Device pinning and multi-GPU + +- **Only `models_16GB_8GB_VRAM.ini` pins devices.** It sets `split-mode`, `main-gpu`, and + `tensor-split` in its `[*]` section; the other two tiers set none of the three, so on a host with + more than one CUDA device llama.cpp's default `split-mode = layer` spreads every entry across + *all* visible GPUs — the tier name is then a floor, not a cap. Pin with `CUDA_VISIBLE_DEVICES` + before launch, not `--device`: in router mode each child's argv is rebuilt from the preset + (`inst.meta.update_args`), so a parent `--device` never reaches the child, while the environment + is copied into every child (`tools/server/server-models.cpp:802`). Pinning is a throughput + decision, not only a memory one — `Ternary-Bonsai-27B-Q2_g64.gguf` measured 56.5 t/s tg and + 1418 t/s pp on one 16 GB card versus 36.5 t/s and 998 t/s spread over a 16 GB plus an 8 GB card. + `CUDA_VISIBLE_DEVICES` indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and + therefore does *not* match `nvidia-smi` ordering; pass a GPU UUID to be unambiguous. + +## load-mode + +- **Every entry but `Qwen3.8-Flash-Next` uses `load-mode = dio`; never pair `direct-io` with + `no-mmap` again.** Both spellings + are deprecated, and they write the *same* mutually exclusive enum — `--no-mmap` sets + `LLAMA_LOAD_MODE_NONE` (`common/arg.cpp:2594`) while `--direct-io` sets + `LLAMA_LOAD_MODE_DIRECT_IO` (`:2603`) — so setting both means only whichever is parsed last wins. + Which one that is is *not* the INI order: `common/preset.cpp` emits `opt.args.back()` while + iterating an unordered map. It happened to resolve to DirectIO, but a container-order change would + silently downgrade to `NONE`, dropping DirectIO *and* mmap for plain buffered reads. `dio` is the + single equivalent of the old pair and also removes two deprecation warnings per launch. Valid + values are `none`, `mmap`, `mlock`, `mmap+mlock`, `dio` (`arg.cpp:2615-2619`) — anything else + throws at startup. + +- **`Qwen3.8-Flash-Next` is the one entry that must use `load-mode = mmap` instead.** Its 26.8 GiB + `per_layer_token_embd` n-gram hash table is created with `TENSOR_READ_LAZY` + (`src/models/qwen4exp.cpp:139-140`) and the loader gates that flag on `use_mmap` + (`src/llama-model-loader.cpp:1290`), which every non-mmap `load-mode` clears (`:559`). A session + touches 16 of the table's 320 million rows per token (`ple_n_heads`, `:64`), so under `mmap` the resident working set + stays in the hundreds of MiB while `dio` reads and holds all 26.8 GiB. `no-host = true` is part + of the same mechanism, not an independent choice — see `docs/model_tuning.md` -> *Qwen3.8-Flash-Next*. + +- **`load-mode = dio` does not enable DirectIO on Windows — it only disables mmap.** The Win32 `llama_file::impl` ctor takes `use_direct_io` as `[[maybe_unused]]` and just calls `ggml_fopen` (`vendor/llama.cpp/src/llama-mmap.cpp:86-95`); `FILE_FLAG_NO_BUFFERING` is never set and `read_alignment()` stays 1 (`:391`), so the loader's async staging buffers are 4 x 1 MiB of pinned host memory instead of the 4 x 64 MiB the aligned path would use (`src/llama-model-loader.cpp:1418`, `:1427`). `has_direct_io()` nevertheless returns a hardcoded `true` on Windows (`:173-175`). Net effect of `dio` on this platform: buffered reads, no mmap, and zero VRAM cost — it is never implicated in a CUDA OOM. Keep the key for the deprecation-warning reason documented above, but do not reason about page-cache behaviour from it. + +## mmproj-offload + +- **`mmproj-offload = true` fails silently at startup on a saturated GPU.** CLIP's warmup + compute buffer OOMs but the server keeps running — only image requests error at generation + time. Set `false` on tiers where LLM + KV already saturate VRAM. + +## Context size and override-kv + +- **`ctx-size` above the GGUF's `context_length` is dead VRAM unless `override-kv` lifts it too.** + `llama-context.cpp:131` never clamps `n_ctx`, so the KV cache really is allocated at the + requested size — but the server then caps every slot at `n_ctx_train` + (`tools/server/server-context.cpp:1209-1214`, applied as `slot.n_ctx = n_ctx_slot` at `:1274`) + and rejects any larger request outright. Both Muse Glimmer GGUFs ship + `muse-glimmer.context_length = 131072`, so the 24 GB entry's former `ctx-size = 262144` + allocated 262144 cells while no request could exceed 131072 — ~884 MiB of unreachable VRAM. + `override-kv = muse-glimmer.context_length=int:262144` raises `n_ctx_train` and is the only + lever; Meta documents 131072 as the default and 262144 as the maximum, so this is the + vendor-sanctioned ceiling, not an extrapolation hack. The startup line + `the slot context (...) exceeds the training context of the model (...) - capping` is expected + here (the pool is 524288 for two 262144 slots); what matters is + `initializing, n_slots = 2, n_ctx_slot = 262144`. If that reads 131072 the override did not + take. It also sets `n_ctx_orig_yarn = 262144` (`src/llama-model.cpp:1180`), inert at + `freq_scale 1`. + +## ngram-mod speculative decoding + +**ngram-mod speculative decoding** (`--spec-type ngram-mod`): model-agnostic, works on any model. +- All models: `spec-ngram-mod-n-match = 24`, `spec-ngram-mod-n-min = 48`, `spec-ngram-mod-n-max = 64` + (matches the struct defaults in `common/common.h:329-337` and what `--spec-default` produces + at `common/arg.cpp:4065-4074`; ggerganov confirmed post-merge in PR #19164 that the min/max + "likely don't need to be changed from the recommended values"; MoEs require long drafts and + dense models tolerate them without noticeable cost). Flags were renamed from + `--draft-min`/`--draft-max`/`--spec-ngram-size-n` in upstream PR #22397; the old names now + error at startup. +- `n_match < 16` logs a "too small — poor quality is possible" warning at + `vendor/llama.cpp/common/speculative.cpp:1031-1034`; parser accepts `1..1024` + (`common/arg.cpp:3606-3615`), so 16 is the lowest non-warning value, not a hard floor. + Min/max parsers accept `0..1024` (`common/arg.cpp:3587-3605`). +- Memory overhead: ~16 MiB **total**, shared across all server slots + (single `common_ngram_mod` instance allocated at `common/speculative.cpp:1026`). +- Pool auto-resets on `begin()` if occupancy > 25 %, and after 3 consecutive rounds with + acceptance < 50 % (`common/speculative.cpp:720-728`, `:790-806`). Smaller `n_match` makes + these resets fire more often and wipes ngrams learned from the current prompt — another + reason to stay at `n_match ≥ 24`. diff --git a/patches/0001-gguf-py-size-row-groups-by-bytes.patch b/patches/0001-gguf-py-size-row-groups-by-bytes.patch new file mode 100644 index 0000000..55acaa6 --- /dev/null +++ b/patches/0001-gguf-py-size-row-groups-by-bytes.patch @@ -0,0 +1,19 @@ +diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py +index 80966b6ef..85c5b49e4 100644 +--- a/gguf-py/gguf/quants.py ++++ b/gguf-py/gguf/quants.py +@@ -33,8 +33,12 @@ def _apply_over_grouped_rows(func: Callable[[np.ndarray], np.ndarray], arr: np.n + for dim in oshape: + osize *= dim + out = np.empty(shape=osize, dtype=otype) +- # compute over groups of 16 rows (arbitrary, but seems good for performance) +- n_groups = (rows.shape[0] // 16) or 1 ++ # grow the group to 160 KiB, but never below 16 rows. a fixed row count ++ # makes the group size follow the row width, which is very slow for some ++ # widths on Windows ++ row_bytes = rows.shape[-1] * arr.dtype.itemsize ++ rows_per_group = max(16, (160 * 1024) // row_bytes) if row_bytes else 16 ++ n_groups = (rows.shape[0] // rows_per_group) or 1 + np.concatenate([func(group).ravel() for group in np.array_split(rows, n_groups)], axis=0, out=out) + return out.reshape(oshape) + diff --git a/presets/README.md b/presets/README.md index f54cf7a..ba8c9e6 100644 --- a/presets/README.md +++ b/presets/README.md @@ -31,7 +31,7 @@ llama-server --models-dir D:\AI\LLM\gguf --models-preset presets\models_16GB_8GB > — they are parent-server settings that the server manages internally and cannot be set via preset. > [!NOTE] -The presets `models_16GB_VRAM.ini`, `models_24GB_VRAM.ini`, and `models_16GB_8GB_VRAM.ini` are each tuned for its VRAM budget (context size, KV quantisation, and MoE offload differ). Copy one as a starting point for other hardware. Only `models_16GB_8GB_VRAM.ini` pins GPUs — the other two leave `split-mode`/`tensor-split` unset, so on a multi-GPU host they spread across every visible CUDA device and may exceed the budget named in the file. Pin with `CUDA_VISIBLE_DEVICES` before launching; its indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and does **not** match `nvidia-smi` ordering, so a GPU UUID is the unambiguous choice. +The presets `models_16GB_VRAM.ini`, `models_24GB_VRAM.ini`, and `models_16GB_8GB_VRAM.ini` are each tuned for its VRAM budget (context size, KV quantisation, and MoE offload differ). Copy one as a starting point for other hardware. Only `models_16GB_8GB_VRAM.ini` pins GPUs — the other two leave `split-mode`/`tensor-split` unset, so on a multi-GPU host they spread across every visible CUDA device and may exceed the budget named in the file. Pin with `CUDA_VISIBLE_DEVICES` before launching; its indices follow `CUDA_DEVICE_ORDER`, which defaults to `FASTEST_FIRST` and does **not** match `nvidia-smi` ordering, so a GPU UUID is the unambiguous choice. Every entry uses `load-mode = dio` except `Qwen3.8-Flash-Next`, which needs `mmap` so that its 26.8 GiB n-gram embedding table can be read on demand — do not normalise that one away. > [!IMPORTANT] > **`models_16GB_8GB_VRAM.ini` (dual-GPU: one GPU with 16 GB VRAM + one with 8 GB VRAM).** @@ -69,14 +69,23 @@ The section header (e.g. `[gemma-4-31B-it.IQ4_XS.gguf]`) is the model name clien > See `llama-server --help` for all flags. > [!IMPORTANT] -> All `Qwen3.6-*` and `Ternary-Bonsai-27B` entries set `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`, +> All `Qwen3.6-*`, `Qwen3.8-*`, `Ternary-Bonsai-27B` and `Bonsai-27B` entries set +> `chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja`, > overriding the buggy template embedded in the GGUF. The vendored template is a -> single unified file that handles both Qwen 3.5 and 3.6 variants; Ternary Bonsai 27B is a -> Qwen3.6-27B derivative with a byte-identical tokenizer and embedded template. The path is +> single unified file that handles Qwen 3.5, 3.6 and 3.8 variants; both Bonsai models are +> Qwen3.6-27B derivatives with a byte-identical tokenizer and embedded template. The path is > repo-relative, so launch `llama-server` from the repository root (as the examples above do). > If you cloned without `--recurse-submodules`, run `git submodule update --init` > first — otherwise startup fails with a missing-file error. > +> The Qwen 3.6 and Bonsai entries additionally set `reasoning-effort = medium`, the `Qwen3.8-*` +> entries `xhigh`. `medium` is the one level that injects no instruction text into the system +> prompt, and Qwen 3.6 has no trained notion of the concept; Qwen 3.8 *is* trained on it and +> `xhigh` is what Qwen's own template defaults to. Both are pinned rather than left unset because +> the vendored template's own default has moved between its releases, so an unpinned entry would +> silently change reasoning level at a template bump. Clients can still override per request via +> the OpenAI `reasoning_effort` field. +> > All `gemma-4-*` entries set `chat-template-file = vendor\llama.cpp\models\templates\google-gemma-4-31B-it.jinja` — > the official Google template bundled with llama.cpp itself, kept in lock-step with > its built-in Gemma 4 parser. The same repo-root launch caveat applies. diff --git a/presets/models_16GB_8GB_VRAM.ini b/presets/models_16GB_8GB_VRAM.ini index 7ae3b58..e8a0008 100644 --- a/presets/models_16GB_8GB_VRAM.ini +++ b/presets/models_16GB_8GB_VRAM.ini @@ -54,6 +54,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -88,6 +89,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -122,6 +124,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 diff --git a/presets/models_16GB_VRAM.ini b/presets/models_16GB_VRAM.ini index a971b29..022e842 100644 --- a/presets/models_16GB_VRAM.ini +++ b/presets/models_16GB_VRAM.ini @@ -16,6 +16,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -49,6 +50,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -82,6 +84,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -115,6 +118,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -147,6 +151,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -177,6 +182,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -190,3 +196,33 @@ spec-ngram-mod-n-min = 48 spec-ngram-mod-n-max = 64 load-mode = dio fit = off + +[Muse-Glimmer-30B.IQ3_XXS.gguf] +alias = Muse-Glimmer-30B.IQ3_XXS.gguf +model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ3_XXS.gguf +mmproj = D:\AI\LLM\gguf\Muse-Glimmer-30B\mmproj.Muse-Glimmer-30B.Q8_0.gguf +spec-draft-model = D:\AI\LLM\gguf\Muse-Glimmer-30B-assistant\dflash-Muse-Glimmer-30B-assistant.Q4_0.gguf +no-mmproj-offload = true +ctx-size = 131072 +parallel = 1 +n-gpu-layers = -1 +cache-type-k = q5_0 +cache-type-v = q4_1 +cache-type-k-draft = q5_0 +cache-type-v-draft = q4_1 +flash-attn = true +jinja = true +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 51200 +min-p = 0.0 +top-p = 0.95 +top-k = 64 +temp = 1.0 +spec-type = draft-dflash,ngram-mod +spec-draft-n-max = 15 +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = dio +fit = off diff --git a/presets/models_24GB_VRAM.ini b/presets/models_24GB_VRAM.ini index 576fb62..51573c4 100644 --- a/presets/models_24GB_VRAM.ini +++ b/presets/models_24GB_VRAM.ini @@ -83,6 +83,7 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -99,10 +100,10 @@ spec-ngram-mod-n-max = 64 load-mode = dio fit = off -[Qwen3.6-35B-A3B.IQ4_XS.gguf] -alias = Qwen3.6-35B-A3B.IQ4_XS.gguf -model = D:\AI\LLM\gguf\Qwen3.6-35B-A3B\Qwen3.6-35B-A3B.IQ4_XS.gguf -mmproj = D:\AI\LLM\gguf\Qwen3.6-35B-A3B\mmproj.Qwen3.6-35B-A3B.BF16.gguf +[Qwen3.8-27B.IQ4_XS.gguf] +alias = Qwen3.8-27B.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Qwen3.8-27B\Qwen3.8-27B.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.8-27B\mmproj.Qwen3.8-27B.Q8_0.gguf mmproj-offload = true image-min-tokens = 1024 ctx-size = 262144 @@ -116,13 +117,14 @@ flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = xhigh kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 min-p = 0.0 top-p = 0.95 top-k = 20 -temp = 0.6 +temp = 1.0 presence-penalty = 0 spec-type = draft-mtp,ngram-mod spec-draft-n-max = 3 @@ -132,22 +134,24 @@ spec-ngram-mod-n-max = 64 load-mode = dio fit = off -[Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf] -alias = Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf -model = D:\AI\LLM\gguf\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.IQ4_XS.gguf -mmproj = D:\AI\LLM\gguf\Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved\mmproj.Qwen3.6-27B-uncensored-heretic-v2-Native-MTP-Preserved.BF16.gguf +[Qwen3.6-35B-A3B.IQ4_XS.gguf] +alias = Qwen3.6-35B-A3B.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Qwen3.6-35B-A3B\Qwen3.6-35B-A3B.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.6-35B-A3B\mmproj.Qwen3.6-35B-A3B.BF16.gguf mmproj-offload = true +image-min-tokens = 1024 ctx-size = 262144 parallel = 1 n-gpu-layers = -1 -cache-type-k = q5_0 -cache-type-v = q4_1 -cache-type-k-draft = q5_0 -cache-type-v-draft = q4_1 +cache-type-k = q4_0 +cache-type-v = q4_0 +cache-type-k-draft = q4_0 +cache-type-v-draft = q4_0 flash-attn = true reasoning = on jinja = true chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = medium kv-unified = true ctx-checkpoints = 32 cache-ram = 51200 @@ -161,11 +165,74 @@ spec-draft-n-max = 3 spec-ngram-mod-n-match = 24 spec-ngram-mod-n-min = 48 spec-ngram-mod-n-max = 64 -direct-io = true -no-mmap = true +load-mode = dio +fit = off + +[Qwen3.8-Flash-Next.IQ4_XS.gguf] +alias = Qwen3.8-Flash-Next.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\Qwen3.8-Flash-Next.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Qwen3.8-Flash-Next\mmproj.Qwen3.8-Flash-Next.Q8_0.gguf +no-mmproj-offload = true +image-min-tokens = 1024 +ctx-size = 524288 +parallel = 2 +n-gpu-layers = -1 +cache-type-k = q8_0 +cache-type-v = q8_0 +flash-attn = true +reasoning = on +jinja = true +chat-template-file = vendor\Qwen-Fixed-Chat-Templates\chat_template.jinja +reasoning-effort = xhigh +kv-unified = true +ctx-checkpoints = 8 +cache-ram = 32768 +min-p = 0.0 +top-p = 0.95 +top-k = 20 +temp = 1.0 +presence-penalty = 0 +spec-type = ngram-mod +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = mmap +no-host = true +fit = on +fit-target = 3072 + +[Muse-Glimmer-30B.IQ4_XS.gguf] +alias = Muse-Glimmer-30B.IQ4_XS.gguf +model = D:\AI\LLM\gguf\Muse-Glimmer-30B\Muse-Glimmer-30B.IQ4_XS.gguf +mmproj = D:\AI\LLM\gguf\Muse-Glimmer-30B\mmproj.Muse-Glimmer-30B.Q8_0.gguf +spec-draft-model = D:\AI\LLM\gguf\Muse-Glimmer-30B-assistant\dflash-Muse-Glimmer-30B-assistant.Q4_0.gguf +mmproj-offload = true +ctx-size = 524288 +override-kv = muse-glimmer.context_length=int:262144 +parallel = 2 +n-gpu-layers = -1 +cache-type-k = q5_0 +cache-type-v = q4_1 +cache-type-k-draft = q5_0 +cache-type-v-draft = q4_1 +flash-attn = true +jinja = true +kv-unified = true +ctx-checkpoints = 32 +cache-ram = 51200 +min-p = 0.0 +top-p = 0.95 +top-k = 64 +temp = 1.0 +spec-type = draft-dflash,ngram-mod +spec-draft-n-max = 15 +spec-ngram-mod-n-match = 24 +spec-ngram-mod-n-min = 48 +spec-ngram-mod-n-max = 64 +load-mode = dio fit = off -[DeepSeek-V4-Flash-0731-UD-Q8_K_XL] +[DeepSeek-V4-Flash-0731-UD-Q8_K_XL.gguf] alias = DeepSeek-V4-Flash-0731-UD-Q8_K_XL.gguf model = D:\AI\LLM\gguf\DeepSeek-V4-Flash-0731-GGUF\UD-Q8_K_XL\DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00001-of-00005.gguf ctx-size = 1048576 diff --git a/rebuild_llama.cpp.ps1 b/rebuild_llama.cpp.ps1 index 944a384..2133bae 100644 --- a/rebuild_llama.cpp.ps1 +++ b/rebuild_llama.cpp.ps1 @@ -251,6 +251,38 @@ if (!$pullRequest) { git -C ./vendor/llama.cpp reset --hard PR } +# Local fixes to the vendored tree, re-applied on every build because the +# submodule was reset to origin/master above. Must run after the checkout, +# otherwise the checkout discards them again. +$patches = @(Get-ChildItem -Path "./patches/*.patch" -ErrorAction "SilentlyContinue" | Sort-Object -Property "Name") + +if ($patches.Count -eq 0) { + Write-Host "[Patches] No patches found under ./patches." -ForegroundColor "DarkYellow" +} + +foreach ($patch in $patches) { + + Write-Host "[Patches] Applying $($patch.Name)..." -ForegroundColor "Yellow" + + # --3way still applies when upstream moved the surrounding lines. A failure + # here means upstream moved the patched code itself, so fail loudly instead + # of silently building an unpatched tree. + git -C ./vendor/llama.cpp apply --3way --whitespace=nowarn $patch.FullName + + if ($LASTEXITCODE -ne 0) { + throw "Failed to apply ./patches/$($patch.Name).`nUpstream likely moved the code it targets. Review and regenerate the patch." + } +} + +if ($patches.Count -gt 0) { + + # `git apply --3way` implies `--index` and would leave the patched files + # staged, which makes `git checkout -- ` restore the patched copy + # instead of the upstream one. Unstage so the submodule shows the same + # plain worktree modification as the OpenBLAS shim below. + git -C ./vendor/llama.cpp reset --quiet +} + $lines = @( "# This is a workaround for a CMake bug on Windows to build llama.cpp" "# with OpenBLAS. The find_package(BLAS) call fails to find OpenBLAS," diff --git a/vendor/Qwen-Fixed-Chat-Templates b/vendor/Qwen-Fixed-Chat-Templates index c31fd39..e649070 160000 --- a/vendor/Qwen-Fixed-Chat-Templates +++ b/vendor/Qwen-Fixed-Chat-Templates @@ -1 +1 @@ -Subproject commit c31fd393e531dbacd92b6deb99a2037cc949f950 +Subproject commit e6490706c74d558c2020b9360e556451488a348b diff --git a/vendor/llama.cpp b/vendor/llama.cpp index e031d95..6c84c7d 160000 --- a/vendor/llama.cpp +++ b/vendor/llama.cpp @@ -1 +1 @@ -Subproject commit e031d956797f9a10f704fa523fe5fd9b72e9015c +Subproject commit 6c84c7d5d8833c6e0df69628f75a0f599797934e