From de96de5df247cfb7eb7fb4646bf06ca835bafb87 Mon Sep 17 00:00:00 2001 From: Alex TYRODE Date: Sat, 25 Jul 2026 02:23:11 +0000 Subject: [PATCH 1/5] feat(generate): rank tiers by capability, not price MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Input cost stopped being a proxy for capability once providers began repricing new models below predecessors they never delisted. Run against today's real catalog, the old scaffolder produced: plan/slow/designer/reviewer -> claude-opus-4-1:high ($15, 200k, xhigh) fallback -> claude-opus-5:high ($5, 1M, max) the smart rung led by a fossil its own fallback dominates on every axis. At the top dial position it also capped thinking, emitting `claude-opus-4-1:xhigh` while the model behind it could do `max`. The ladder is now derived rather than guessed from price: - supersession first: only the newest member of each model family survives, so claude-opus-4-1 never competes with claude-opus-5. Version components compare numerically, so a future 5.10 beats 5.9. - ranking by thinking ceiling, then context, then price. Price is the last word, not the first: among models with identical published specs it is the only remaining signal of size, but across generations it lies. - loadCatalog now asserts the property the ranking should produce. A dearer rung offering less context or less thinking headroom is rejected with both models named, rather than silently shipped. Tiers 0 and 4 were previously unreachable — pickLadder only ever emitted tiers 1..3, so the spark and fable dials were dead on every scaffolded config and the shipped default (spark on) opened the TUI on a combo that was never generated. They are now read from the tier-scoped quota buckets omp already reports, capped at one per pool and sanity-checked against price. Also: - --bench fills speed/ttft from `omp bench --json` instead of writing 50/2.0 for every model, which made the speed meter model-invariant. It doubles as a reachability probe: omp lists claude-mythos-5 at claude-fable-5's exact price but it 404s here, and no metadata distinguishes them. A failed probe drops the model. - --refresh re-derives tiers over an existing file. init used to refuse outright, which is how a catalog goes three model generations stale without anyone noticing. - thinking is a level set, not a range. claude-opus-4-6 offers low/medium/high/max but not xhigh; a range claimed a level the API rejects. - same-family short keys disambiguate by version (opus5, opus48) rather than by ladder index, which named the newer model `opus` and the older one `opus3`. The models file gains bucket, context and image fields. The init fixture now models the catalog's real shape — four same-priced Opus variants, an undelisted $15 legacy, two identically priced elites, a text-only spark — so it can actually fail on the bug it is meant to guard. --- generate.go | 207 +++++++++++++--- generate_init.go | 597 +++++++++++++++++++++++++++++++++++++++++---- generate_test.go | 523 ++++++++++++++++++++++++++++++++++----- onboarding.go | 9 +- onboarding_test.go | 3 +- 5 files changed, 1186 insertions(+), 153 deletions(-) diff --git a/generate.go b/generate.go index 82f169b..0f2f0dd 100644 --- a/generate.go +++ b/generate.go @@ -4,7 +4,7 @@ package main // browses (see loadBlocks) from a models file, and `code generate init` // scaffolds that models file from the user's own omp instance. // -// This is the Go port of the dotfiles' generate-profiles.py (atyrode/dotfiles, +// This began as a Go port of the dotfiles' generate-profiles.py (atyrode/dotfiles, // pkgs/omp-configured), generalised from that setup's hard-coded model keys to // pure pool/tier logic so it works against anyone's catalog: // @@ -13,16 +13,16 @@ package main // providers are present; generation fails loudly otherwise. // - tier 0 (an idle-bucket speed model, "spark") and tier 4 (a scarce elite, // "fable") are optional; without them the corresponding facet combos are -// simply not generated and the TUI reports "no profile for this -// combination" when dialed there. +// simply not generated and the TUI hides the dial. // -// The output format is byte-compatible with the Python generator so a catalog -// produced by either renders identically in the TUI. +// The dotfiles build now invokes this binary rather than keeping its own copy +// of the renderer, so this file is the single source of the catalog format. import ( "fmt" "os" "path/filepath" + "sort" "strings" "gopkg.in/yaml.v3" @@ -31,19 +31,29 @@ import ( // ── model catalog ───────────────────────────────────────────────────────────── type catModel struct { - ID string `yaml:"id"` - Pool string `yaml:"pool"` - Tier int `yaml:"tier"` - CostIn float64 `yaml:"cost_in"` - CostOut float64 `yaml:"cost_out"` - Speed float64 `yaml:"speed"` - TTFT float64 `yaml:"ttft"` - Thinking string `yaml:"thinking"` + ID string `yaml:"id"` + Pool string `yaml:"pool"` + Tier int `yaml:"tier"` + Bucket string `yaml:"bucket"` + CostIn float64 `yaml:"cost_in"` + CostOut float64 `yaml:"cost_out"` + Speed float64 `yaml:"speed"` + TTFT float64 `yaml:"ttft"` + Context int `yaml:"context"` + // "lo→hi" for the usual contiguous case, or a comma list when the model + // has a hole in the scale (see parseThinkingLevels). + Thinking string `yaml:"thinking"` + // Absent means "accepts images" — true of every model in both pools today + // except the codex spark variants, which `generate init` marks explicitly. + Image *bool `yaml:"image"` } +func (m catModel) multimodal() bool { return m.Image == nil || *m.Image } + type catalog struct { keys []string // declaration order (drives __models__ rows) models map[string]catModel // short key -> model + levels map[string][]int // short key -> thinking levels it truly offers ladder map[string][5]string // ladder[pool][tier] for tiers 0..4; "" = absent. Tiers 1..3 are the // fallback ladder; 0 is the drain/speed lead, 4 the elite lead. @@ -90,7 +100,7 @@ func loadCatalogBytes(raw []byte, path string) (*catalog, error) { if modelsNode == nil { return nil, fmt.Errorf("%s: no `models:` mapping", path) } - c := &catalog{models: map[string]catModel{}, ladder: map[string][5]string{"O": {}, "A": {}}} + c := &catalog{models: map[string]catModel{}, levels: map[string][]int{}, ladder: map[string][5]string{"O": {}, "A": {}}} for i := 0; i+1 < len(modelsNode.Content); i += 2 { key := modelsNode.Content[i].Value var m catModel @@ -103,11 +113,13 @@ func loadCatalogBytes(raw []byte, path string) (*catalog, error) { if m.Tier < 0 || m.Tier > 4 { return nil, fmt.Errorf("%s: model %q: tier must be 0..4, got %d", path, key, m.Tier) } - if _, _, err := parseThinkingRange(m.Thinking); err != nil { + levels, err := parseThinkingLevels(m.Thinking) + if err != nil { return nil, fmt.Errorf("%s: model %q: %w", path, key, err) } c.keys = append(c.keys, key) c.models[key] = m + c.levels[key] = levels l := c.ladder[m.Pool] if l[m.Tier] != "" { return nil, fmt.Errorf("%s: pool %s tier %d claimed by both %q and %q", path, m.Pool, m.Tier, l[m.Tier], key) @@ -122,33 +134,104 @@ func loadCatalogBytes(raw []byte, path string) (*catalog, error) { } } } + if err := c.checkLadder(path); err != nil { + return nil, err + } return c, nil } -func parseThinkingRange(s string) (lo, hi int, err error) { +// checkLadder rejects a ladder whose rungs are out of order. Input price used +// to be a fair proxy for capability, so the scaffolder ranked by it — then +// providers started repricing new models below predecessors they never +// delisted, and a $15 claude-opus-4-1 (200k context, no max thinking) outranked +// a $5 claude-opus-5 (1M, max) on price alone. Rather than trust the ranking, +// assert the property it is supposed to produce: a dearer rung must not offer +// less. Only tiers 1..3 are the capability ladder — tier 0 is a bucket-drain +// lead and tier 4 an elite lead, both deliberately off it. +func (c *catalog) checkLadder(path string) error { + for _, pool := range []string{"O", "A"} { + for lo := 1; lo <= 3; lo++ { + for hi := lo + 1; hi <= 3; hi++ { + a, b := c.ladder[pool][lo], c.ladder[pool][hi] + if a == "" || b == "" { + continue + } + if why := c.regression(a, b); why != "" { + return fmt.Errorf("%s: pool %s tier %d (%s) is a regression on tier %d (%s): %s — reorder the tiers, or drop the superseded model and re-run `code generate init --refresh`", + path, pool, hi, c.models[b].ID, lo, c.models[a].ID, why) + } + } + } + } + return nil +} + +// regression reports why rung hi is worse than the cheaper rung lo, or "" when +// it isn't. A pricier model with more context and more thinking headroom is the +// ladder working; a pricier model with less of either is a stale pick. +func (c *catalog) regression(lo, hi string) string { + l, h := c.models[lo], c.models[hi] + if h.CostIn < l.CostIn { + return "" // cheaper up the ladder is odd but not a capability loss + } + if l.Context > 0 && h.Context > 0 && h.Context < l.Context { + return fmt.Sprintf("%d context at $%s/1M vs %d at $%s", h.Context, trimFloat(h.CostIn), l.Context, trimFloat(l.CostIn)) + } + ll, hl := c.levels[lo], c.levels[hi] + if hl[len(hl)-1] < ll[len(ll)-1] { + return fmt.Sprintf("thinking tops out at %s vs %s, and costs $%s/1M vs $%s", thScale[hl[len(hl)-1]], thScale[ll[len(ll)-1]], trimFloat(h.CostIn), trimFloat(l.CostIn)) + } + return "" +} + +// parseThinkingLevels resolves a thinking declaration to the sorted set of +// levels the model truly offers. Two forms: "lo→hi" for the usual contiguous +// run, and a comma list for a model with a hole in the scale — claude-opus-4-6 +// offers low/medium/high/max but not xhigh, and a range would claim a level the +// API rejects. +func parseThinkingLevels(s string) ([]int, error) { + if strings.Contains(s, ",") { + var out []int + for _, part := range strings.Split(s, ",") { + name := strings.TrimSpace(part) + i, ok := thIdx(name) + if !ok { + return nil, fmt.Errorf("unknown thinking level %q in %q", name, s) + } + out = append(out, i) + } + sort.Ints(out) + return out, nil + } parts := strings.Split(s, "→") if len(parts) != 2 { - return 0, 0, fmt.Errorf("thinking must be \"lo→hi\" (e.g. low→max), got %q", s) + return nil, fmt.Errorf("thinking must be \"lo→hi\" or a comma list (e.g. low→max, or low,medium,high,max), got %q", s) } lo, okLo := thIdx(strings.TrimSpace(parts[0])) hi, okHi := thIdx(strings.TrimSpace(parts[1])) if !okLo || !okHi || lo > hi { - return 0, 0, fmt.Errorf("invalid thinking range %q", s) + return nil, fmt.Errorf("invalid thinking range %q", s) } - return lo, hi, nil + out := make([]int, 0, hi-lo+1) + for i := lo; i <= hi; i++ { + out = append(out, i) + } + return out, nil } -// clampTh resolves a requested level to one the model actually offers. +// clampTh resolves a requested level to the nearest one the model actually +// offers, rounding down so a dial never buys more thinking than was asked for. +// A request below the model's floor takes the floor. func (c *catalog) clampTh(key, level string) string { - lo, hi, _ := parseThinkingRange(c.models[key].Thinking) + levels := c.levels[key] i, _ := thIdx(level) - if i < lo { - i = lo - } - if i > hi { - i = hi + best := levels[0] + for _, l := range levels { + if l <= i { + best = l + } } - return thScale[i] + return thScale[best] } func otherPool(p string) string { @@ -214,18 +297,32 @@ func (c *catalog) buildChain(lead string, isPure bool) []string { return dedup([]string{sib, cr, c.sibDown(cr)}, lead) } +// visionLead is the cheapest rung on a pool that accepts image input. The +// codex spark variants are text-only, so the cheapest rung is not always it. +func (c *catalog) visionLead(pool string) string { + for t := 1; t <= 3; t++ { + if k := c.ladder[pool][t]; k != "" && c.models[k].multimodal() { + return k + } + } + return "" +} + // ── the facet grid ──────────────────────────────────────────────────────────── var ( genRoleOrder = []string{"default", "task", "plan", "slow", "designer", "reviewer", - "librarian", "sonic", "advisor", "smol", "tiny", "commit"} - genAgentRoles = map[string]bool{"designer": true, "librarian": true, "reviewer": true, "sonic": true, "task": true} - genDelib = map[string]bool{"plan": true, "slow": true, "designer": true, "reviewer": true} + "librarian", "scout", "sonic", "advisor", "vision", "smol", "tiny", "commit"} + // The six agents omp bundles. scout was the one this grid never routed, so + // it silently inherited @smol and never appeared in the preview. + genAgentRoles = map[string]bool{"designer": true, "librarian": true, "reviewer": true, + "scout": true, "sonic": true, "task": true} + genDelib = map[string]bool{"plan": true, "slow": true, "designer": true, "reviewer": true} // Anti-tunnel-vision: on a *-led lane the reviewer crosses to the opposite // provider so the output always gets an independent second eye (the advisor // crosses too, in its own branch). genCrossLed = map[string]bool{"reviewer": true} - genUtil = map[string]bool{"sonic": true, "smol": true, "tiny": true, "commit": true} + genUtil = map[string]bool{"scout": true, "sonic": true, "smol": true, "tiny": true, "commit": true} // Utility roles respond to the dials but are tier-capped so none can ever // become expensive. genUtilModel = map[string]map[string]int{ @@ -233,12 +330,16 @@ var ( "tiny": {"fast": 1, "normal": 1, "smart": 2}, "smol": {"fast": 1, "normal": 2, "smart": 2}, "sonic": {"fast": 1, "normal": 2, "smart": 2}, + "scout": {"fast": 1, "normal": 2, "smart": 2}, } genUtilThink = map[string]map[string]string{ "commit": {"low": "minimal", "medium": "minimal", "high": "minimal", "xhigh": "low"}, "tiny": {"low": "minimal", "medium": "low", "high": "low", "xhigh": "low"}, "smol": {"low": "low", "medium": "low", "high": "medium", "xhigh": "medium"}, "sonic": {"low": "low", "medium": "medium", "high": "medium", "xhigh": "medium"}, + // omp's bundled scout runs at medium; it reads broadly, so it keeps a + // step more thinking than smol at the same rung. + "scout": {"low": "low", "medium": "medium", "high": "medium", "xhigh": "medium"}, } genTierMap = map[string]int{"fast": 1, "normal": 2, "smart": 3} genBump = map[string]string{"minimal": "low", "low": "medium", "medium": "high", "high": "xhigh", "xhigh": "xhigh"} @@ -312,7 +413,9 @@ func (c *catalog) genCombo(lane, mtier, thinking string, spark, fable, fableMain fb = []string{c.ladder[rp][t]} } else { lead = c.ladder[rp][t] - if r == "sonic" { // only sonic keeps a net + // scout and sonic are spawned constantly and block their caller, + // so they keep a net; the rest are cheap enough to just retry. + if r == "sonic" || r == "scout" { if sd := c.sibDown(lead); sd != "" { fb = []string{sd} } @@ -322,6 +425,33 @@ func (c *catalog) genCombo(lane, mtier, thinking string, spark, fable, fableMain out[r] = roleRoute{lead, th, chain, repeatLvl(th, len(chain))} continue } + if r == "vision" { + // omp falls back @vision → @default → active model when it needs an + // image described, and describeForTextModels is on by default — so + // this rung must be a model that actually accepts images. + lead := c.visionLead(p) + if lead == "" && !isPure { + lead = c.visionLead(otherPool(p)) + } + if lead == "" { + out[r] = roleRoute{} + continue + } + // Describing an image is not a reasoning task; keep it cheap unless + // the dial sits at an extreme, where the operator asked for uniformity. + th := "low" + if extreme { + th = thinking + } + var chain []string + for _, k := range c.buildChain(lead, isPure) { + if c.models[k].multimodal() { + chain = append(chain, k) + } + } + out[r] = roleRoute{lead, th, chain, repeatLvl(th, len(chain))} + continue + } if r == "advisor" { if mtier == "fast" { out[r] = roleRoute{} @@ -455,12 +585,19 @@ func (c *catalog) renderCombo(lane, mtier, thinking string, spark, fable, fableM return strings.Join(lines, "\n") } +// renderModelFacts emits the per-model table the TUI's meters read. The bucket +// is a trailing optional column: the consumer falls back to guessing from the +// model family when a catalog omits it, so old catalogs keep working. func (c *catalog) renderModelFacts() string { - lines := []string{"__models__ model facts (id in out speed ttft — $/1M in·out, tok/s, s)"} + lines := []string{"__models__ model facts (id in out speed ttft bucket — $/1M in·out, tok/s, s)"} for _, k := range c.keys { m := c.models[k] - lines = append(lines, fmt.Sprintf(" %s %s %s %s %s", - m.ID, trimFloat(m.CostIn), trimFloat(m.CostOut), trimFloat(m.Speed), trimFloat(m.TTFT))) + row := fmt.Sprintf(" %s %s %s %s %s", + m.ID, trimFloat(m.CostIn), trimFloat(m.CostOut), trimFloat(m.Speed), trimFloat(m.TTFT)) + if m.Bucket != "" { + row += " " + m.Bucket + } + lines = append(lines, row) } lines = append(lines, "") return strings.Join(lines, "\n") diff --git a/generate_init.go b/generate_init.go index 63baa78..7009826 100644 --- a/generate_init.go +++ b/generate_init.go @@ -9,11 +9,13 @@ package main import ( "encoding/json" "fmt" + "math" "os" "os/exec" "path/filepath" "regexp" "sort" + "strconv" "strings" ) @@ -23,6 +25,7 @@ type ompModel struct { ContextWindow int `json:"contextWindow"` Reasoning bool `json:"reasoning"` Thinking []string `json:"thinking"` + Input []string `json:"input"` // modalities; "image" gates the vision role Cost struct { Input float64 `json:"input"` Output float64 `json:"output"` @@ -51,58 +54,379 @@ func poolOf(provider string) string { return "" } +var versionToken = regexp.MustCompile(`^[\d.]+$`) + // shortKey derives a memorable key from a model id: the last dash-separated // token that isn't purely a version (e.g. claude-sonnet-5 → sonnet, -// gpt-5.6-terra → terra, gpt-5.3-codex-spark → spark). +// gpt-5.6-terra → terra, gpt-5.3-codex-spark → spark). Same-family ids collapse +// to the same key, which scaffoldModels resolves with versionSuffix. func shortKey(id string) string { toks := strings.Split(id, "-") for i := len(toks) - 1; i >= 0; i-- { - if !regexp.MustCompile(`^[\d.]+$`).MatchString(toks[i]) { + if !versionToken.MatchString(toks[i]) { return strings.ToLower(toks[i]) } } return strings.ToLower(strings.NewReplacer(".", "-", ":", "-").Replace(id)) } -// pickLadder guesses tiers 1..3 for one pool: candidates ranked by input cost, -// cheapest → tier 1, priciest → tier 3, the distinct cost nearest the middle → -// tier 2. Same-cost ties prefer the larger context window, then the shorter id -// (the canonical alias rather than a dated variant). +// versionSuffix compacts an id's version tokens for disambiguating two models +// of one family: claude-opus-5 → "5", claude-opus-4-8 → "48". The old +// disambiguator was the ladder index, which named the newer model `opus` and +// the older one `opus3` — no help at all to someone hand-editing the file. +func versionSuffix(id string) string { + var b strings.Builder + for _, t := range strings.Split(id, "-") { + if versionToken.MatchString(t) { + b.WriteString(strings.ReplaceAll(t, ".", "")) + } + } + return b.String() +} + +// familyOf splits an id into its name tokens and its version components: +// claude-opus-4-8 → "claude-opus", [4 8]; gpt-5.6-terra → "gpt-terra", [5 6]. +// Two ids share a family when their name tokens match, which is what makes +// "claude-opus-5 supersedes claude-opus-4-8" decidable without a curated list. +// Dotted tokens split into separate components rather than parsing as a float, +// so a future gpt-5.10 outranks gpt-5.9 instead of reading as 5.1. +func familyOf(id string) (string, []int) { + var name []string + var ver []int + for _, t := range strings.Split(id, "-") { + if versionToken.MatchString(t) { + parts := strings.Split(t, ".") + nums := make([]int, 0, len(parts)) + for _, p := range parts { + n, err := strconv.Atoi(p) + if err != nil { + nums = nil + break + } + nums = append(nums, n) + } + if nums != nil { + ver = append(ver, nums...) + continue + } + } + name = append(name, strings.ToLower(t)) + } + return strings.Join(name, "-"), ver +} + +// newer compares version vectors component-wise; a longer vector wins only +// where it agrees on every shared component (4.8 beats 4.1, and 5 beats 4.8). +func newer(a, b []int) bool { + for i := range min(len(a), len(b)) { + if a[i] != b[i] { + return a[i] > b[i] + } + } + return len(a) > len(b) +} + +// supersede keeps only the newest member of each family. Providers reprice new +// models below predecessors they never delist — claude-opus-4-1 still lists at +// $15/1M while its successor claude-opus-5 costs $5 — so a price-ranked ladder +// crowns the fossil. Dropping superseded siblings removes the trap at source. +func supersede(cands []ompModel) []ompModel { + type entry struct { + m ompModel + fam string + ver []int + } + es := make([]entry, 0, len(cands)) + for _, m := range cands { + fam, ver := familyOf(m.ID) + es = append(es, entry{m, fam, ver}) + } + sort.Slice(es, func(i, j int) bool { + if es[i].fam != es[j].fam { + return es[i].fam < es[j].fam + } + if newer(es[i].ver, es[j].ver) { + return true + } + if newer(es[j].ver, es[i].ver) { + return false + } + return es[i].m.ID < es[j].m.ID + }) + var out []ompModel + seen := map[string]bool{} + for _, e := range es { + if seen[e.fam] { + continue + } + seen[e.fam] = true + out = append(out, e.m) + } + return out +} + +// ceiling is the highest thinking level a model offers — the capability signal +// omp exposes that, unlike price, does not go stale. +func ceiling(m ompModel) int { + hi := -1 + for _, lv := range m.Thinking { + if i, ok := thIdx(lv); ok && i > hi { + hi = i + } + } + return hi +} + +// moreCapable ranks by thinking ceiling, then context, then price. Price is the +// last word rather than the first: among models with identical published specs +// it is the only remaining signal of size, but across generations it lies. +func moreCapable(a, b ompModel) bool { + if ceiling(a) != ceiling(b) { + return ceiling(a) > ceiling(b) + } + if a.ContextWindow != b.ContextWindow { + return a.ContextWindow > b.ContextWindow + } + if a.Cost.Input != b.Cost.Input { + return a.Cost.Input > b.Cost.Input + } + return a.ID < b.ID +} + +// cheaper orders by price, most capable first within a price, then by id so the +// result never depends on sort stability. +func cheaper(a, b ompModel) bool { + if a.Cost.Input != b.Cost.Input { + return a.Cost.Input < b.Cost.Input + } + if ceiling(a) != ceiling(b) { + return ceiling(a) > ceiling(b) + } + if a.ContextWindow != b.ContextWindow { + return a.ContextWindow > b.ContextWindow + } + return a.ID < b.ID +} + +// pickLadder chooses tiers 1..3 for one pool from an already-superseded +// candidate set: cheapest is tier 1, the most capable is tier 3, and tier 2 is +// the most capable model priced strictly between them. func pickLadder(cands []ompModel) []ompModel { - if len(cands) == 0 { + if len(cands) < 3 { + return cands + } + sort.Slice(cands, func(i, j int) bool { return cheaper(cands[i], cands[j]) }) + t1, rest := cands[0], cands[1:] + t3 := rest[0] + for _, m := range rest[1:] { + if moreCapable(m, t3) { + t3 = m + } + } + t2, found := ompModel{}, false + for _, m := range rest { + if m.ID == t3.ID || m.Cost.Input <= t1.Cost.Input || m.Cost.Input >= t3.Cost.Input { + continue + } + if !found || moreCapable(m, t2) { + t2, found = m, true + } + } + if !found { // no room between the rungs — take the median by price + t2 = rest[len(rest)/2] + if t2.ID == t3.ID { + t2 = rest[0] + } + } + return []ompModel{t1, t2, t3} +} + +// ompUsageJSON fetches the provider quota report; a var so tests and the +// onboarding flow can stub it. Failure is never fatal — the special tiers it +// feeds are optional. +var ompUsageJSON = func() ([]byte, error) { + return exec.Command("omp", "usage", "--json").Output() +} + +// specialTier is a quota bucket omp scopes to a subset of a provider's models: +// the codex spark drain bucket, and Anthropic's scarce elite bucket. omp names +// these itself, so the scaffolder reads them rather than guessing which model +// is the cheap drain and which is the scarce elite — the guess used to be +// "nobody, ever", which left the spark and fable dials dead on every scaffold. +type specialTier struct { + pool, tier, modelID string +} + +func readSpecialTiers() []specialTier { + raw, err := ompUsageJSON() + if err != nil { + return nil + } + var u struct { + Reports []struct { + Provider string `json:"provider"` + Limits []struct { + Scope struct { + Provider string `json:"provider"` + Tier string `json:"tier"` + ModelID string `json:"modelId"` + } `json:"scope"` + } `json:"limits"` + } `json:"reports"` + } + if json.Unmarshal(raw, &u) != nil { return nil } - sort.Slice(cands, func(i, j int) bool { - if cands[i].Cost.Input != cands[j].Cost.Input { - return cands[i].Cost.Input < cands[j].Cost.Input + var out []specialTier + seen := map[string]bool{} + for _, r := range u.Reports { + for _, l := range r.Limits { + if l.Scope.Tier == "" { + continue + } + pool := poolOf(l.Scope.Provider) + if pool == "" { + pool = poolOf(r.Provider) + } + if pool == "" || seen[pool+l.Scope.Tier] { + continue + } + seen[pool+l.Scope.Tier] = true + out = append(out, specialTier{pool, l.Scope.Tier, l.Scope.ModelID}) } - if cands[i].ContextWindow != cands[j].ContextWindow { - return cands[i].ContextWindow > cands[j].ContextWindow + } + return out +} + +// matchSpecial resolves a tier-scoped bucket to one of the pool's candidates. A +// scope that names its model wins outright; otherwise the bucket's tier name is +// matched against the model family, which is where the label comes from (omp's +// "Claude 7 Day (Fable)" is the claude-fable-* family). Matching by name and +// not by price matters: claude-mythos-5 lists at claude-fable-5's exact price +// but is not served on every account. +func matchSpecial(st specialTier, cands []ompModel) string { + if st.modelID != "" { + for _, m := range cands { + if strings.EqualFold(m.ID, st.modelID) { + return m.ID + } } - return len(cands[i].ID) < len(cands[j].ID) - }) - // first model per distinct cost - var distinct []ompModel - seen := map[float64]bool{} + } for _, m := range cands { - if !seen[m.Cost.Input] { - seen[m.Cost.Input] = true - distinct = append(distinct, m) + if fam, _ := familyOf(m.ID); strings.Contains(fam, strings.ToLower(st.tier)) { + return m.ID } } - switch len(distinct) { - case 1: - return distinct - case 2: - return distinct - default: - return []ompModel{distinct[0], distinct[len(distinct)/2], distinct[len(distinct)-1]} + return "" +} + +// bucketName follows the catalog's existing convention: the pool's own quota +// window, or a tier-scoped one when the model draws a separate bucket. +func bucketName(pool, tier string) string { + base := "codex" + if pool == "A" { + base = "claude" + } + if tier == "" { + return base + "-main" } + return base + "-" + tier +} + +// thinkingField renders a model's levels for models.yml: the compact "lo→hi" +// range when the levels are contiguous, and an explicit comma list when they +// are not. claude-opus-4-6 offers low/medium/high/max but not xhigh, and a +// range would promise a level the API rejects. +func thinkingField(levels []string) string { + idx := make([]int, 0, len(levels)) + for _, lv := range levels { + if i, ok := thIdx(lv); ok { + idx = append(idx, i) + } + } + sort.Ints(idx) + contiguous := true + for i := 1; i < len(idx); i++ { + if idx[i] != idx[i-1]+1 { + contiguous = false + break + } + } + if contiguous { + return thScale[idx[0]] + "→" + thScale[idx[len(idx)-1]] + } + names := make([]string, len(idx)) + for i, v := range idx { + names[i] = thScale[v] + } + return strings.Join(names, ",") +} + +// benchFact is one model's measured throughput, or a recorded probe failure. +type benchFact struct { + speed, ttft float64 + ok bool +} + +// ompBenchJSON measures models with omp's own benchmark; a var so tests can +// stub it. Three runs is enough to smooth a cold start without turning +// `--bench` into a coffee break. +var ompBenchJSON = func(selectors []string) ([]byte, error) { + args := append([]string{"bench"}, selectors...) + return exec.Command("omp", append(args, "--json", "--runs", "3")...).Output() +} + +// runBench fills in speed and ttft, and doubles as a reachability probe. omp's +// catalog lists models the account cannot actually call — claude-mythos-5 is +// priced identically to claude-fable-5 and 404s here — and no metadata +// distinguishes them, so a model whose probe fails is recorded as unreachable +// and dropped from the ladder rather than merely left unmeasured. +func runBench(selectors []string) (map[string]benchFact, error) { + raw, err := ompBenchJSON(selectors) + if len(raw) == 0 { + if err == nil { + err = fmt.Errorf("no output") + } + return nil, fmt.Errorf("running `omp bench`: %w", err) + } + var parsed struct { + Models []struct { + Model string `json:"model"` + Average *struct { + TTFTMs float64 `json:"ttftMs"` + TokensPerSecond float64 `json:"tokensPerSecond"` + } `json:"average"` + } `json:"models"` + } + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("parsing bench report: %w", err) + } + out := map[string]benchFact{} + for _, m := range parsed.Models { + id := m.Model + if i := strings.LastIndexByte(id, '/'); i >= 0 { + id = id[i+1:] + } + // omp is not consistent about id casing across surfaces (usage scopes + // report "GPT-5.3-Codex-Spark"), so key on a folded id at both ends. + id = strings.ToLower(id) + if m.Average == nil { + out[id] = benchFact{} + continue + } + out[id] = benchFact{ + speed: math.Round(m.Average.TokensPerSecond*10) / 10, + ttft: math.Round(m.Average.TTFTMs/10) / 100, + ok: true, + } + } + return out, nil } // scaffoldModels turns an `omp models --json` payload into models.yml content. -// Pure, so the CLI and the first-run onboarding share it. -func scaffoldModels(raw []byte) (string, error) { +// Pure but for the optional quota probe, so the CLI and the first-run +// onboarding share it. bench, when non-nil, supplies measured speed/ttft. +func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { var parsed ompModels if err := json.Unmarshal(raw, &parsed); err != nil { return "", fmt.Errorf("parsing model list: %w", err) @@ -114,9 +438,15 @@ func scaffoldModels(raw []byte) (string, error) { m.Cost.Input <= 0 || datedID.MatchString(m.ID) { continue } + // A model omp listed but could not actually call is worse than useless + // on the ladder: it looks top-spec and fails at launch. Only --bench + // knows, so only --bench can filter. + if f, probed := bench[strings.ToLower(m.ID)]; probed && !f.ok { + continue + } // Keep only thinking levels the generator's scale knows (omp can expose // provider-specific extras like "off"); without this the scaffold would - // write a range 'code generate' rejects one step later. + // write a level 'code generate' rejects one step later. var levels []string for _, lv := range m.Thinking { if _, ok := thIdx(lv); ok { @@ -129,55 +459,176 @@ func scaffoldModels(raw []byte) (string, error) { m.Thinking = levels byPool[pool] = append(byPool[pool], m) } - ladders := map[string][]ompModel{"O": pickLadder(byPool["O"]), "A": pickLadder(byPool["A"])} - for pool, name := range map[string]string{"O": "OpenAI/Codex", "A": "Anthropic"} { - if len(ladders[pool]) < 3 { - return "", fmt.Errorf("found %d usable %s model(s), need 3 (cheap/regular/smart) — code assumes both Anthropic and OpenAI are set up in omp", len(ladders[pool]), name) + + specials := readSpecialTiers() + type rung struct { + m ompModel + tier int + bucket string + } + rungs := map[string][]rung{} + for _, pool := range []string{"O", "A"} { + cands := supersede(byPool[pool]) + // Lift a tier-scoped model out before ranking: it is a lead, not a rung + // on the capability ladder. At most one per pool — two would both claim + // the same tier and loadCatalog would refuse the file — preferring the + // bucket whose scope names its model, then the first by name so the + // choice never depends on report ordering. + var pick *specialTier + var pickID string + for i := range specials { + st := &specials[i] + if st.pool != pool { + continue + } + id := matchSpecial(*st, cands) + if id == "" { + continue + } + if pick == nil || (st.modelID != "" && pick.modelID == "") || + (st.modelID != "" == (pick.modelID != "") && st.tier < pick.tier) { + pick, pickID = st, id + } } + // The grid reads tier 0 off pool O and tier 4 off pool A, so the pool + // decides which kind of lead this is — but only price confirms it. An + // elite is the scarce top of its pool, a drain bucket is not; a scoped + // bucket that fits neither shape is some other quota window and is left + // on the ordinary ladder. + var top float64 + for _, m := range cands { + if m.ID != pickID && m.Cost.Input > top { + top = m.Cost.Input + } + } + specialTierNo := -1 + if pick != nil { + c := cands[0].Cost.Input + for _, m := range cands { + if m.ID == pickID { + c = m.Cost.Input + } + } + if pool == "A" && c >= top { + specialTierNo = 4 + } else if pool == "O" && c < top { + specialTierNo = 0 + } + } + var elite ompModel + var ladderCands []ompModel + for _, m := range cands { + if specialTierNo < 0 || m.ID != pickID { + ladderCands = append(ladderCands, m) + continue + } + if specialTierNo == 4 { + elite = m + } + rungs[pool] = append(rungs[pool], rung{m, specialTierNo, bucketName(pool, pick.tier)}) + } + // An elite defines the scarce, expensive class. Anything priced at or + // above it is a sibling elite rather than a tier-3 workhorse, and must + // not be crowned "smart" — claude-mythos-5 is exactly that trap: same + // price as claude-fable-5, and 404 on accounts that do not have it. + if elite.ID != "" { + var kept []ompModel + for _, m := range ladderCands { + if m.Cost.Input < elite.Cost.Input { + kept = append(kept, m) + } + } + ladderCands = kept + } + ladder := pickLadder(ladderCands) + if len(ladder) < 3 { + name := "OpenAI/Codex" + if pool == "A" { + name = "Anthropic" + } + return "", fmt.Errorf("found %d usable %s model(s), need 3 (cheap/regular/smart) — code assumes both Anthropic and OpenAI are set up in omp", len(ladder), name) + } + for i, m := range ladder { + rungs[pool] = append(rungs[pool], rung{m, i + 1, bucketName(pool, "")}) + } + sort.Slice(rungs[pool], func(i, j int) bool { return rungs[pool][i].tier < rungs[pool][j].tier }) } var b strings.Builder b.WriteString(`# Model catalog for code's routing generator — scaffolded by 'code generate init'. # -# REVIEW THIS FILE. The ids, costs, and thinking ranges come from your omp; -# the tier assignments are auto-guessed from price and the speed/ttft numbers -# are placeholder estimates (they only drive the TUI's speed meter). +# REVIEW THIS FILE. The ids, costs, context and thinking levels come from your +# omp; the tier assignments are derived (newest model per family, then ranked by +# thinking ceiling, context and price) and worth a sanity check. # # pool: O = OpenAI/Codex · A = Anthropic # tier: 1 cheap · 2 regular · 3 smart (the per-pool fallback ladder) -# Optional extras: tier 0 on pool O = a fast idle-bucket model the -# 'spark' toggle drains; tier 4 on pool A = a scarce elite the -# 'fable' toggle leads with. Add them if you have such models. +# tier 0 = a fast idle-bucket model the 'spark' toggle drains; +# tier 4 = a scarce elite the 'fable' toggle leads with. Both are +# detected from the quota buckets omp reports, and simply absent when +# your providers expose no such bucket. +# bucket: the quota window the model draws — drives the usage meter. +# image: false marks a text-only model, which the vision role then avoids. # # Re-render the catalog after any edit: code generate +# Re-derive the tiers after a provider ships new models: code generate init --refresh models: `) used := map[string]bool{} for _, pool := range []string{"O", "A"} { - for i, m := range ladders[pool] { - key := shortKey(m.ID) + for _, r := range rungs[pool] { + key := shortKey(r.m.ID) + if used[key] { + key += versionSuffix(r.m.ID) + } for used[key] { - key += fmt.Sprintf("%d", i+1) + key += "x" } used[key] = true + speed, ttft := "50", "2.0" + note := " # placeholder — run `code generate init --refresh --bench` to measure" + if f, ok := bench[strings.ToLower(r.m.ID)]; ok { + speed, ttft, note = trimFloat(f.speed), trimFloat(f.ttft), "" + } b.WriteString(fmt.Sprintf(` %s: id: %s pool: %s tier: %d + bucket: %s cost_in: %s cost_out: %s - speed: 50 # placeholder — measured tok/s if you have it - ttft: 2.0 # placeholder — measured seconds to first token - thinking: %s→%s -`, key, m.ID, pool, i+1, trimFloat(m.Cost.Input), trimFloat(m.Cost.Output), - m.Thinking[0], m.Thinking[len(m.Thinking)-1])) + speed: %s%s + ttft: %s%s + context: %d + thinking: %s +`, key, r.m.ID, pool, r.tier, r.bucket, trimFloat(r.m.Cost.Input), trimFloat(r.m.Cost.Output), + speed, note, ttft, note, r.m.ContextWindow, thinkingField(r.m.Thinking))) + if !imageCapable(r.m) { + b.WriteString(" image: false\n") + } } } return b.String(), nil } +// imageCapable reports whether omp lists image input for the model. Absent +// input data is treated as capable: every model in both pools accepts images +// today except the codex spark variants. +func imageCapable(m ompModel) bool { + if len(m.Input) == 0 { + return true + } + for _, in := range m.Input { + if in == "image" { + return true + } + } + return false +} + func runGenerateInit(args []string) int { fromJSON, out := "", defaultModelsPath() + bench, refresh := false, false for i := 0; i < len(args); i++ { switch args[i] { case "--from-json": @@ -194,6 +645,10 @@ func runGenerateInit(args []string) int { return 2 } out = args[i] + case "--bench": + bench = true + case "--refresh": + refresh = true case "-h", "--help": fmt.Print(generateHelp) return 0 @@ -218,7 +673,23 @@ func runGenerateInit(args []string) int { fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) return 1 } - yml, err := scaffoldModels(raw) + + var facts map[string]benchFact + if bench { + sels, err := benchSelectors(raw) + if err != nil { + fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) + return 1 + } + fmt.Fprintf(os.Stderr, "benchmarking %d models — this makes real API calls and takes a while…\n", len(sels)) + facts, err = runBench(sels) + if err != nil { + fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) + return 1 + } + } + + yml, err := scaffoldModels(raw, facts) if err != nil { fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) return 1 @@ -228,14 +699,36 @@ func runGenerateInit(args []string) int { fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) return 1 } - if _, err := os.Stat(out); err == nil { - fmt.Fprintf(os.Stderr, "code generate init: %s already exists — review or delete it first\n", out) + if _, err := os.Stat(out); err == nil && !refresh { + fmt.Fprintf(os.Stderr, "code generate init: %s already exists — pass --refresh to re-derive the tiers from your current model list, or delete it first\n", out) return 1 } if err := os.WriteFile(out, []byte(yml), 0o644); err != nil { fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) return 1 } - fmt.Printf("wrote %s — review the tier guesses, then run `code generate`\n", out) + verb := "wrote" + if refresh { + verb = "refreshed" + } + fmt.Printf("%s %s — review the tiers, then run `code generate`\n", verb, out) return 0 } + +// benchSelectors lists every model the scaffolder would consider, as +// provider-qualified selectors so `omp bench` cannot fuzzy-match the wrong one. +func benchSelectors(raw []byte) ([]string, error) { + var parsed ompModels + if err := json.Unmarshal(raw, &parsed); err != nil { + return nil, fmt.Errorf("parsing model list: %w", err) + } + var out []string + for _, m := range parsed.Models { + if poolOf(m.Provider) == "" || !m.Reasoning || m.Cost.Input <= 0 || datedID.MatchString(m.ID) { + continue + } + out = append(out, m.Provider+"/"+m.ID) + } + sort.Strings(out) + return out, nil +} diff --git a/generate_test.go b/generate_test.go index fee5835..080f48b 100644 --- a/generate_test.go +++ b/generate_test.go @@ -1,98 +1,121 @@ package main import ( + "fmt" "os" "path/filepath" "strings" "testing" ) -// fixtureYML mirrors the reference catalog the Go port was byte-parity -// verified against (the Python generator's output on the same input matched -// renderCatalog exactly, diff-clean, 2026-07-16). The golden strings below -// come from that verified output. +// fixtureYML mirrors the shape of a real catalog: both pools filled, the +// optional spark/elite tiers present, an explicit quota bucket per model, and a +// text-only model (the codex spark variants really are text-only) so the vision +// role has something to route around. const fixtureYML = `models: luna: id: gpt-5.6-luna pool: O tier: 1 + bucket: codex-main cost_in: 1 cost_out: 6 speed: 52.3 ttft: 1.18 + context: 272000 thinking: low→max terra: id: gpt-5.6-terra pool: O tier: 2 + bucket: codex-main cost_in: 2.5 cost_out: 15 speed: 51.8 ttft: 1.74 + context: 272000 thinking: low→max sol: id: gpt-5.6-sol pool: O tier: 3 + bucket: codex-main cost_in: 5 cost_out: 30 speed: 31.5 ttft: 4.59 + context: 272000 thinking: low→max spark: id: gpt-5.3-codex-spark pool: O tier: 0 + bucket: codex-spark cost_in: 1.75 cost_out: 14 speed: 286.7 ttft: 5.56 + context: 128000 thinking: low→xhigh + image: false haiku: id: claude-haiku-4-5 pool: A tier: 1 + bucket: claude-main cost_in: 1 cost_out: 5 speed: 48.9 ttft: 1.7 + context: 200000 thinking: minimal→xhigh sonnet: id: claude-sonnet-5 pool: A tier: 2 + bucket: claude-main cost_in: 2 cost_out: 10 speed: 35.2 ttft: 3.84 + context: 1000000 thinking: low→max opus: - id: claude-opus-4-8 + id: claude-opus-5 pool: A tier: 3 + bucket: claude-main cost_in: 5 cost_out: 25 speed: 46.6 ttft: 1.77 + context: 1000000 thinking: low→max fable: id: claude-fable-5 pool: A tier: 4 + bucket: claude-fable cost_in: 10 cost_out: 50 speed: 54 ttft: 6.9 + context: 1000000 thinking: low→max ` -func fixtureCatalog(t *testing.T) *catalog { +func catalogFrom(t *testing.T, yml string) (*catalog, error) { t.Helper() p := filepath.Join(t.TempDir(), "models.yml") - if err := os.WriteFile(p, []byte(fixtureYML), 0o644); err != nil { + if err := os.WriteFile(p, []byte(yml), 0o644); err != nil { t.Fatal(err) } - c, err := loadCatalog(p) + return loadCatalog(p) +} + +func fixtureCatalog(t *testing.T) *catalog { + t.Helper() + c, err := catalogFrom(t, fixtureYML) if err != nil { t.Fatalf("loadCatalog: %v", err) } @@ -105,20 +128,34 @@ const goldenAdvisors = `__advisors__ advisor dial (level context → chain) audit gpt gpt-5.6-sol:high → gpt-5.6-terra:high → gpt-5.6-luna:low glance claude claude-haiku-4-5:low review claude claude-sonnet-5:medium → claude-haiku-4-5:low - audit claude claude-opus-4-8:high → claude-sonnet-5:high → claude-haiku-4-5:low + audit claude claude-opus-5:high → claude-sonnet-5:high → claude-haiku-4-5:low +` + +// goldenFacts pins the trailing bucket column the TUI's quota meter reads. +const goldenFacts = `__models__ model facts (id in out speed ttft bucket — $/1M in·out, tok/s, s) + gpt-5.6-luna 1 6 52.3 1.18 codex-main + gpt-5.6-terra 2.5 15 51.8 1.74 codex-main + gpt-5.6-sol 5 30 31.5 4.59 codex-main + gpt-5.3-codex-spark 1.75 14 286.7 5.56 codex-spark + claude-haiku-4-5 1 5 48.9 1.7 claude-main + claude-sonnet-5 2 10 35.2 3.84 claude-main + claude-opus-5 5 25 46.6 1.77 claude-main + claude-fable-5 10 50 54 6.9 claude-fable ` const goldenMixedSmart = `mixed_smart_medium_sp_fa mixed · smart · medium · spark · fable thinking medium · fallback on · advisor on - default gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-4-8:medium → claude-sonnet-5:medium - ● task gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-4-8:medium → claude-sonnet-5:medium - plan claude-fable-5:high → claude-opus-4-8:high → gpt-5.6-sol:high → gpt-5.6-terra:high - slow claude-fable-5:high → claude-opus-4-8:high → gpt-5.6-sol:high → gpt-5.6-terra:high - ● designer claude-fable-5:high → claude-opus-4-8:high → gpt-5.6-sol:high → gpt-5.6-terra:high - ● reviewer claude-fable-5:high → claude-opus-4-8:high → gpt-5.6-sol:high → gpt-5.6-terra:high - ● librarian gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-4-8:medium → claude-sonnet-5:medium + default gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-5:medium → claude-sonnet-5:medium + ● task gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-5:medium → claude-sonnet-5:medium + plan claude-fable-5:high → claude-opus-5:high → gpt-5.6-sol:high → gpt-5.6-terra:high + slow claude-fable-5:high → claude-opus-5:high → gpt-5.6-sol:high → gpt-5.6-terra:high + ● designer claude-fable-5:high → claude-opus-5:high → gpt-5.6-sol:high → gpt-5.6-terra:high + ● reviewer claude-fable-5:high → claude-opus-5:high → gpt-5.6-sol:high → gpt-5.6-terra:high + ● librarian gpt-5.6-sol:medium → gpt-5.6-terra:medium → claude-opus-5:medium → claude-sonnet-5:medium + ● scout gpt-5.6-terra:medium → gpt-5.6-luna:medium ● sonic gpt-5.6-terra:medium → gpt-5.6-luna:medium advisor claude-sonnet-5:high → claude-haiku-4-5:low → gpt-5.6-terra:low → gpt-5.6-luna:low + vision gpt-5.6-luna:low → claude-haiku-4-5:low smol gpt-5.6-terra:low tiny gpt-5.3-codex-spark:low → gpt-5.6-terra:low commit gpt-5.3-codex-spark:low → gpt-5.6-luna:low @@ -126,42 +163,97 @@ const goldenMixedSmart = `mixed_smart_medium_sp_fa mixed · smart · medium · const goldenClaudeMax = `claude-only_normal_max_nosp_famain claude-only · normal · max · fable · main thinking max · fallback on · advisor on - default claude-fable-5:max → claude-opus-4-8:max → claude-sonnet-5:max + default claude-fable-5:max → claude-opus-5:max → claude-sonnet-5:max ● task claude-sonnet-5:max → claude-haiku-4-5:xhigh - plan claude-fable-5:max → claude-opus-4-8:max → claude-sonnet-5:max - slow claude-fable-5:max → claude-opus-4-8:max → claude-sonnet-5:max - ● designer claude-fable-5:max → claude-opus-4-8:max → claude-sonnet-5:max - ● reviewer claude-fable-5:max → claude-opus-4-8:max → claude-sonnet-5:max + plan claude-fable-5:max → claude-opus-5:max → claude-sonnet-5:max + slow claude-fable-5:max → claude-opus-5:max → claude-sonnet-5:max + ● designer claude-fable-5:max → claude-opus-5:max → claude-sonnet-5:max + ● reviewer claude-fable-5:max → claude-opus-5:max → claude-sonnet-5:max ● librarian claude-sonnet-5:max → claude-haiku-4-5:xhigh + ● scout claude-sonnet-5:max → claude-haiku-4-5:xhigh ● sonic claude-sonnet-5:max → claude-haiku-4-5:xhigh advisor claude-haiku-4-5:xhigh + vision claude-haiku-4-5:xhigh smol claude-sonnet-5:max tiny claude-haiku-4-5:xhigh commit claude-haiku-4-5:xhigh ` +// goldenClaudeSmart is the one combo where the Anthropic smart rung is itself +// the padded lead column, so a change to that model's id shifts the padding +// rather than just swapping a chain token. Every other golden here has a pool-O +// model or the elite in the lead. +const goldenClaudeSmart = `claude-only_smart_medium_nosp_nofa claude-only · smart · medium + thinking medium · fallback on · advisor on + default claude-opus-5:medium → claude-sonnet-5:medium → claude-haiku-4-5:medium + ● task claude-opus-5:medium → claude-sonnet-5:medium → claude-haiku-4-5:medium + plan claude-opus-5:high → claude-sonnet-5:high → claude-haiku-4-5:high + slow claude-opus-5:high → claude-sonnet-5:high → claude-haiku-4-5:high + ● designer claude-opus-5:high → claude-sonnet-5:high → claude-haiku-4-5:high + ● reviewer claude-opus-5:high → claude-sonnet-5:high → claude-haiku-4-5:high + ● librarian claude-opus-5:medium → claude-sonnet-5:medium → claude-haiku-4-5:medium + ● scout claude-sonnet-5:medium → claude-haiku-4-5:medium + ● sonic claude-sonnet-5:medium → claude-haiku-4-5:medium + advisor claude-sonnet-5:high → claude-haiku-4-5:low + vision claude-haiku-4-5:low + smol claude-sonnet-5:low + tiny claude-sonnet-5:low + commit claude-haiku-4-5:minimal +` + func TestGoldenAdvisors(t *testing.T) { c := fixtureCatalog(t) if got := c.renderAdvisors(); got != goldenAdvisors { - t.Errorf("renderAdvisors mismatch:\n--- got ---\n%s--- want ---\n%s", got, goldenAdvisors) + t.Errorf("advisors mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, goldenAdvisors) } } -func TestGoldenCombos(t *testing.T) { +func TestGoldenModelFacts(t *testing.T) { c := fixtureCatalog(t) - if got := c.renderCombo("mixed", "smart", "medium", true, true, false); got != goldenMixedSmart { - t.Errorf("mixed_smart_medium_sp_fa mismatch:\n--- got ---\n%s--- want ---\n%s", got, goldenMixedSmart) + if got := c.renderModelFacts(); got != goldenFacts { + t.Errorf("model facts mismatch:\n--- got ---\n%s\n--- want ---\n%s", got, goldenFacts) + } +} + +// A catalog that declares no buckets keeps the old five-column rows, so the +// consumer's fallback path stays exercised. +func TestModelFactsWithoutBuckets(t *testing.T) { + c, err := catalogFrom(t, strings.ReplaceAll(fixtureYML, " bucket: codex-main\n", "")) + if err != nil { + t.Fatalf("loadCatalog: %v", err) + } + if !strings.Contains(c.renderModelFacts(), " gpt-5.6-luna 1 6 52.3 1.18\n") { + t.Errorf("bucketless model row should stop after ttft:\n%s", c.renderModelFacts()) } - if got := c.renderCombo("claude-only", "normal", "max", false, true, true); got != goldenClaudeMax { - t.Errorf("claude-only_normal_max_nosp_famain mismatch:\n--- got ---\n%s--- want ---\n%s", got, goldenClaudeMax) +} + +func TestGoldenCombos(t *testing.T) { + c := fixtureCatalog(t) + for _, tc := range []struct { + name, want string + render func() string + }{ + {"mixed_smart_medium_sp_fa", goldenMixedSmart, func() string { + return c.renderCombo("mixed", "smart", "medium", true, true, false) + }}, + {"claude-only_normal_max_nosp_famain", goldenClaudeMax, func() string { + return c.renderCombo("claude-only", "normal", "max", false, true, true) + }}, + {"claude-only_smart_medium_nosp_nofa", goldenClaudeSmart, func() string { + return c.renderCombo("claude-only", "smart", "medium", false, false, false) + }}, + } { + if got := tc.render(); got != tc.want { + t.Errorf("%s mismatch:\n--- got ---\n%s\n--- want ---\n%s", tc.name, got, tc.want) + } } } func TestRenderCatalogStructure(t *testing.T) { c := fixtureCatalog(t) out := c.renderCatalog() - // 414 combos on the full fixture — the count the verified reference - // produced (5 lanes × 3 tiers × 6 thinking × spark/fable/main validity). + // 414 combos on the full fixture: 5 lanes × 3 tiers × 6 thinking levels, + // times the spark/fable/main combinations genValid admits. combos := 0 for _, l := range strings.Split(out, "\n") { if l != "" && l[0] != ' ' && strings.Contains(l, "_") && !strings.HasPrefix(l, "__") { @@ -201,6 +293,47 @@ func TestRenderCatalogStructure(t *testing.T) { walk(0) } +// Every role the generator emits must be weighted, or weightedModels drops it +// from both meters without a trace. +func TestEveryEmittedRoleIsWeighted(t *testing.T) { + for _, r := range genRoleOrder { + if _, ok := roleWeight[r]; !ok { + t.Errorf("role %q is emitted but has no roleWeight entry", r) + } + } +} + +// scout is one of omp's six bundled agents; it must carry the agent marker so +// genConfigYAML mirrors it into task.agentModelOverrides. +func TestScoutIsAgentBacked(t *testing.T) { + c := fixtureCatalog(t) + block := c.renderCombo("mixed", "normal", "medium", false, false, false) + if !strings.Contains(block, "● scout ") { + t.Errorf("scout must render as an agent-backed role:\n%s", block) + } +} + +// The vision role feeds omp's image-describe fallback, so it must never lead on +// a text-only model even when that model is the cheapest rung available. +func TestVisionSkipsTextOnlyModels(t *testing.T) { + // Promote the text-only spark to tier 1 and demote luna out of the way. + yml := strings.Replace(fixtureYML, " id: gpt-5.6-luna\n pool: O\n tier: 1", " id: gpt-5.6-luna\n pool: O\n tier: 0", 1) + yml = strings.Replace(yml, " id: gpt-5.3-codex-spark\n pool: O\n tier: 0", " id: gpt-5.3-codex-spark\n pool: O\n tier: 1", 1) + c, err := catalogFrom(t, yml) + if err != nil { + t.Fatalf("loadCatalog: %v", err) + } + if lead := c.visionLead("O"); lead == "" || c.models[lead].ID != "gpt-5.6-terra" { + t.Errorf("visionLead(O) = %q, want the cheapest image-capable rung (terra)", lead) + } + block := c.renderCombo("gpt-only", "fast", "medium", false, false, false) + for _, l := range strings.Split(block, "\n") { + if strings.Contains(l, " vision ") && strings.Contains(l, "codex-spark") { + t.Errorf("vision must not route to a text-only model: %s", l) + } + } +} + func TestCatalogWithoutOptionalTiers(t *testing.T) { trimmed := "" skip := false @@ -216,11 +349,7 @@ func TestCatalogWithoutOptionalTiers(t *testing.T) { trimmed += line + "\n" } } - p := filepath.Join(t.TempDir(), "models.yml") - if err := os.WriteFile(p, []byte(trimmed), 0o644); err != nil { - t.Fatal(err) - } - c, err := loadCatalog(p) + c, err := catalogFrom(t, trimmed) if err != nil { t.Fatalf("loadCatalog without tier 0/4: %v", err) } @@ -239,18 +368,43 @@ func TestLoadCatalogValidation(t *testing.T) { "bad pool": strings.Replace(fixtureYML, "pool: O", "pool: X", 1), "dup tier": strings.Replace(fixtureYML, "tier: 2", "tier: 1", 1), "bad thinking": strings.Replace(fixtureYML, "low→max", "low-max", 1), + "unknown level": strings.Replace(fixtureYML, "thinking: low→max", "thinking: low,medium,enormous", 1), } for name, yml := range cases { - p := filepath.Join(t.TempDir(), name+".yml") - if err := os.WriteFile(p, []byte(yml), 0o644); err != nil { - t.Fatal(err) - } - if _, err := loadCatalog(p); err == nil { + if _, err := catalogFrom(t, yml); err == nil { t.Errorf("%s: expected an error", name) } } } +// A dearer rung offering less context or less thinking headroom is the exact +// shape a price-ranked scaffold used to produce, so the catalog refuses it. +func TestLadderRegressionRejected(t *testing.T) { + cases := map[string]string{ + "smaller context at a higher price": strings.Replace(fixtureYML, + " id: claude-opus-5\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 5\n cost_out: 25\n speed: 46.6\n ttft: 1.77\n context: 1000000\n thinking: low→max\n", + " id: claude-opus-4-1\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 15\n cost_out: 75\n speed: 46.6\n ttft: 1.77\n context: 200000\n thinking: minimal→xhigh\n", 1), + "lower thinking ceiling at a higher price": strings.Replace(fixtureYML, + " id: claude-opus-5\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 5\n cost_out: 25\n speed: 46.6\n ttft: 1.77\n context: 1000000\n thinking: low→max\n", + " id: claude-opus-4-1\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 15\n cost_out: 75\n speed: 46.6\n ttft: 1.77\n context: 1000000\n thinking: low→xhigh\n", 1), + } + for name, yml := range cases { + _, err := catalogFrom(t, yml) + if err == nil { + t.Errorf("%s: expected the ladder check to reject this catalog", name) + continue + } + if !strings.Contains(err.Error(), "regression") { + t.Errorf("%s: error should name the regression, got %v", name, err) + } + } + // The healthy fixture must not trip it: tier 3 costs more than tier 2 while + // matching it on context and thinking, which is the ladder working. + if _, err := catalogFrom(t, fixtureYML); err != nil { + t.Errorf("healthy ladder rejected: %v", err) + } +} + func TestClampTh(t *testing.T) { c := fixtureCatalog(t) for _, tc := range [][3]string{ @@ -265,6 +419,38 @@ func TestClampTh(t *testing.T) { } } +// A hole in the middle of a model's scale must not be smoothed over: asking for +// the missing level rounds down to one the model really offers. claude-opus-4-6 +// is the live example — low, medium, high, max, but no xhigh. +func TestClampThHonoursGaps(t *testing.T) { + c, err := catalogFrom(t, strings.Replace(fixtureYML, + " id: claude-opus-5\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 5\n cost_out: 25\n speed: 46.6\n ttft: 1.77\n context: 1000000\n thinking: low→max\n", + " id: claude-opus-4-6\n pool: A\n tier: 3\n bucket: claude-main\n cost_in: 5\n cost_out: 25\n speed: 46.6\n ttft: 1.77\n context: 1000000\n thinking: low,medium,high,max\n", 1)) + if err != nil { + t.Fatalf("comma-list thinking should load: %v", err) + } + for _, tc := range [][2]string{{"xhigh", "high"}, {"max", "max"}, {"minimal", "low"}, {"medium", "medium"}} { + if got := c.clampTh("opus", tc[0]); got != tc[1] { + t.Errorf("clampTh(opus, %s) = %s, want %s", tc[0], got, tc[1]) + } + } + if strings.Contains(c.renderCatalog(), "claude-opus-4-6:xhigh") { + t.Error("generator emitted a thinking level the model does not offer") + } +} + +func TestThinkingField(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"low medium high xhigh max", "low→max"}, + {"low medium high max", "low,medium,high,max"}, + {"minimal", "minimal→minimal"}, + } { + if got := thinkingField(strings.Fields(tc.in)); got != tc.want { + t.Errorf("thinkingField(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + func TestTrimFloat(t *testing.T) { for f, want := range map[float64]string{1: "1", 2.5: "2.5", 52.3: "52.3", 286.7: "286.7", 0.25: "0.25"} { if got := trimFloat(f); got != want { @@ -273,53 +459,264 @@ func TestTrimFloat(t *testing.T) { } } +func TestFamilyAndSupersede(t *testing.T) { + for _, tc := range []struct { + id, fam string + ver []int + }{ + {"claude-opus-4-8", "claude-opus", []int{4, 8}}, + {"claude-opus-5", "claude-opus", []int{5}}, + {"gpt-5.6-terra", "gpt-terra", []int{5, 6}}, + {"gpt-5.4-mini", "gpt-mini", []int{5, 4}}, + {"gpt-5.5", "gpt", []int{5, 5}}, + } { + fam, ver := familyOf(tc.id) + if fam != tc.fam || fmt.Sprint(ver) != fmt.Sprint(tc.ver) { + t.Errorf("familyOf(%s) = %q %v, want %q %v", tc.id, fam, ver, tc.fam, tc.ver) + } + } + // Version components compare numerically, not as a decimal: 5.10 is newer + // than 5.9 even though 5.1 < 5.9. + if _, a := familyOf("gpt-5.10"); func() bool { _, b := familyOf("gpt-5.9"); return !newer(a, b) }() { + t.Error("gpt-5.10 must supersede gpt-5.9") + } + mk := func(id string, cost float64) ompModel { + m := ompModel{ID: id, Provider: "anthropic", Reasoning: true, Thinking: []string{"low", "max"}} + m.Cost.Input = cost + return m + } + // The whole point: the $15 fossil never reaches the ladder. + got := supersede([]ompModel{mk("claude-opus-4-1", 15), mk("claude-opus-5", 5), mk("claude-opus-4-8", 5), mk("claude-haiku-4-5", 1)}) + var ids []string + for _, m := range got { + ids = append(ids, m.ID) + } + want := "claude-haiku-4-5 claude-opus-5" + if strings.Join(ids, " ") != want { + t.Errorf("supersede = %v, want %q", ids, want) + } +} + +func TestShortKeyDisambiguation(t *testing.T) { + for _, tc := range [][2]string{ + {"claude-opus-5", "opus"}, + {"claude-opus-4-8", "opus"}, + {"gpt-5.6-sol", "sol"}, + {"gpt-5.3-codex-spark", "spark"}, + } { + if got := shortKey(tc[0]); got != tc[1] { + t.Errorf("shortKey(%s) = %s, want %s", tc[0], got, tc[1]) + } + } + // Same-family ids collide on the short key; the version breaks the tie in a + // way a reader can actually interpret. + for _, tc := range [][2]string{{"claude-opus-5", "5"}, {"claude-opus-4-8", "48"}, {"gpt-5.6-terra", "56"}} { + if got := versionSuffix(tc[0]); got != tc[1] { + t.Errorf("versionSuffix(%s) = %s, want %s", tc[0], got, tc[1]) + } + } +} + +// initJSON models the real catalog's awkward shape: four same-priced Opus +// variants, a legacy Opus that never got delisted and still lists at 3x the +// price, two identically priced elites, a text-only spark, dated snapshots, a +// non-reasoning model, and a free local model. const initJSON = `{"models":[ - {"provider":"anthropic","id":"claude-haiku-4-5","contextWindow":200000,"reasoning":true,"thinking":["minimal","low","medium","high","xhigh"],"cost":{"input":1,"output":5}}, - {"provider":"anthropic","id":"claude-sonnet-5","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"cost":{"input":2,"output":10}}, - {"provider":"anthropic","id":"claude-opus-4-8","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"cost":{"input":5,"output":25}}, - {"provider":"anthropic","id":"claude-opus-4-5-20251101","contextWindow":200000,"reasoning":true,"thinking":["low","high"],"cost":{"input":5,"output":25}}, - {"provider":"anthropic","id":"claude-3-sonnet-20240229","contextWindow":200000,"reasoning":false,"thinking":null,"cost":{"input":3,"output":15}}, - {"provider":"openai-codex","id":"gpt-5.6-luna","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"cost":{"input":1,"output":6}}, - {"provider":"openai-codex","id":"gpt-5.6-terra","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"cost":{"input":2.5,"output":15}}, - {"provider":"openai-codex","id":"gpt-5.6-sol","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"cost":{"input":5,"output":30}}, - {"provider":"ollama","id":"qwen2.5:3b","contextWindow":32768,"reasoning":false,"thinking":null,"cost":{"input":0,"output":0}} + {"provider":"anthropic","id":"claude-haiku-4-5","contextWindow":200000,"reasoning":true,"thinking":["minimal","low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":1,"output":5}}, + {"provider":"anthropic","id":"claude-sonnet-4-5","contextWindow":1000000,"reasoning":true,"thinking":["minimal","low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":3,"output":15}}, + {"provider":"anthropic","id":"claude-sonnet-5","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":2,"output":10}}, + {"provider":"anthropic","id":"claude-opus-4-5","contextWindow":200000,"reasoning":true,"thinking":["minimal","low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":5,"output":25}}, + {"provider":"anthropic","id":"claude-opus-4-6","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","max"],"input":["text","image"],"cost":{"input":5,"output":25}}, + {"provider":"anthropic","id":"claude-opus-4-8","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":5,"output":25}}, + {"provider":"anthropic","id":"claude-opus-5","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":5,"output":25}}, + {"provider":"anthropic","id":"claude-opus-4-1","contextWindow":200000,"reasoning":true,"thinking":["minimal","low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":15,"output":75}}, + {"provider":"anthropic","id":"claude-fable-5","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":10,"output":50}}, + {"provider":"anthropic","id":"claude-mythos-5","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":10,"output":50}}, + {"provider":"anthropic","id":"claude-opus-4-5-20251101","contextWindow":200000,"reasoning":true,"thinking":["low","high"],"input":["text","image"],"cost":{"input":5,"output":25}}, + {"provider":"anthropic","id":"claude-3-sonnet-20240229","contextWindow":200000,"reasoning":false,"thinking":null,"input":["text","image"],"cost":{"input":3,"output":15}}, + {"provider":"openai-codex","id":"gpt-5.4-mini","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":0.75,"output":4.5}}, + {"provider":"openai-codex","id":"gpt-5.6-luna","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":1,"output":6}}, + {"provider":"openai-codex","id":"gpt-5.3-codex-spark","contextWindow":128000,"reasoning":true,"thinking":["low","medium","high","xhigh"],"input":["text"],"cost":{"input":1.75,"output":14}}, + {"provider":"openai-codex","id":"gpt-5.6-terra","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":2.5,"output":15}}, + {"provider":"openai-codex","id":"gpt-5.4","contextWindow":1000000,"reasoning":true,"thinking":["low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":2.5,"output":15}}, + {"provider":"openai-codex","id":"gpt-5.5","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh"],"input":["text","image"],"cost":{"input":5,"output":30}}, + {"provider":"openai-codex","id":"gpt-5.6-sol","contextWindow":272000,"reasoning":true,"thinking":["low","medium","high","xhigh","max"],"input":["text","image"],"cost":{"input":5,"output":30}}, + {"provider":"ollama","id":"qwen2.5:3b","contextWindow":32768,"reasoning":false,"thinking":null,"input":["text"],"cost":{"input":0,"output":0}} ]}` -func TestGenerateInitScaffold(t *testing.T) { +// initUsage is the shape omp reports: a spark bucket that names its model, and +// an elite bucket that only names its tier. +const initUsage = `{"reports":[ + {"provider":"openai-codex","limits":[ + {"id":"openai-codex:primary","scope":{"provider":"openai-codex"}}, + {"id":"openai-codex:spark:primary","scope":{"provider":"openai-codex","tier":"spark","modelId":"GPT-5.3-Codex-Spark"}}]}, + {"provider":"anthropic","limits":[ + {"id":"anthropic:7d","scope":{"provider":"anthropic"}}, + {"id":"anthropic:7d:fable","scope":{"provider":"anthropic","tier":"fable"}}]} +]}` + +// stubOmp points the scaffolder's omp probes at fixtures for the duration of a +// test. usage may be "" to simulate a machine where the probe fails. +func stubOmp(t *testing.T, usage string) { + t.Helper() + prev := ompUsageJSON + ompUsageJSON = func() ([]byte, error) { + if usage == "" { + return nil, fmt.Errorf("no usage") + } + return []byte(usage), nil + } + t.Cleanup(func() { ompUsageJSON = prev }) +} + +func scaffoldTo(t *testing.T, args ...string) (*catalog, string) { + t.Helper() dir := t.TempDir() src := filepath.Join(dir, "omp.json") out := filepath.Join(dir, "models.yml") if err := os.WriteFile(src, []byte(initJSON), 0o644); err != nil { t.Fatal(err) } - if code := runGenerateInit([]string{"--from-json", src, "--models-file", out}); code != 0 { + if code := runGenerateInit(append([]string{"--from-json", src, "--models-file", out}, args...)); code != 0 { t.Fatalf("runGenerateInit exit %d", code) } c, err := loadCatalog(out) if err != nil { - t.Fatalf("scaffold does not load back: %v", err) + body, _ := os.ReadFile(out) + t.Fatalf("scaffold does not load back: %v\n%s", err, body) + } + return c, out +} + +func TestGenerateInitScaffold(t *testing.T) { + stubOmp(t, initUsage) + c, out := scaffoldTo(t) + + // The headline property: the newest model in each family wins its rung, so + // the undelisted $15 claude-opus-4-1 never outranks the $5 claude-opus-5. + want := map[string][5]string{ + "O": {"gpt-5.3-codex-spark", "gpt-5.4-mini", "gpt-5.6-terra", "gpt-5.6-sol", ""}, + "A": {"", "claude-haiku-4-5", "claude-sonnet-5", "claude-opus-5", "claude-fable-5"}, + } + for pool, ids := range want { + for tier, id := range ids { + key := c.ladder[pool][tier] + got := "" + if key != "" { + got = c.models[key].ID + } + if got != id { + t.Errorf("pool %s tier %d = %q, want %q", pool, tier, got, id) + } + } + } + // Two models collapse to the short key "opus" only if a superseded sibling + // survives; none should, so the key stays clean. + if _, ok := c.models["opus"]; !ok { + t.Errorf("expected a clean 'opus' key, got %v", c.keys) + } + + body, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + text := string(body) + // The text-only spark must be marked so the vision role avoids it. + if !strings.Contains(text, "image: false") { + t.Error("scaffold should mark the text-only codex spark") + } + if !strings.Contains(text, "bucket: codex-spark") || !strings.Contains(text, "bucket: claude-fable") { + t.Errorf("scaffold should carry the tier-scoped buckets omp reported:\n%s", text) + } + // A scaffolded catalog must render end-to-end, spark and fable included. + rendered := c.renderCatalog() + for _, want := range []string{"\nmixed_smart_medium_nosp_nofa ", "\nmixed_smart_medium_sp_fa "} { + if !strings.Contains(rendered, want) { + t.Errorf("scaffolded catalog missing %q", want) + } + } +} + +// Without a usage probe the special tiers are simply absent — the scaffold must +// still be a valid, loadable ladder rather than a hard failure. +func TestGenerateInitWithoutUsageProbe(t *testing.T) { + stubOmp(t, "") + c, _ := scaffoldTo(t) + if c.ladder["O"][0] != "" || c.ladder["A"][4] != "" { + t.Errorf("no usage report should mean no special tiers, got %v / %v", c.ladder["O"], c.ladder["A"]) } for _, pool := range []string{"O", "A"} { for tier := 1; tier <= 3; tier++ { if c.ladder[pool][tier] == "" { - t.Errorf("scaffold left pool %s tier %d empty", pool, tier) + t.Errorf("pool %s tier %d left empty", pool, tier) + } + } + } +} + +// --bench doubles as a reachability probe: omp lists claude-mythos-5 at +// claude-fable-5's exact price, but it 404s on accounts that do not have it, +// and no metadata distinguishes the two. +func TestGenerateInitBenchDropsUnreachableModels(t *testing.T) { + stubOmp(t, "") + prev := ompBenchJSON + ompBenchJSON = func(sels []string) ([]byte, error) { + var rows []string + for _, s := range sels { + id := s[strings.LastIndexByte(s, '/')+1:] + if id == "claude-mythos-5" { + rows = append(rows, fmt.Sprintf(`{"model":%q,"average":null}`, s)) + continue } + rows = append(rows, fmt.Sprintf(`{"model":%q,"average":{"ttftMs":1404.2,"tokensPerSecond":48.94}}`, s)) } + return []byte(`{"models":[` + strings.Join(rows, ",") + `]}`), nil } - // Cost-ranked guesses on this fixture: cheapest→1, priciest→3; dated and - // non-reasoning variants excluded. - if c.models[c.ladder["A"][1]].ID != "claude-haiku-4-5" || c.models[c.ladder["A"][3]].ID != "claude-opus-4-8" { - t.Errorf("unexpected A ladder: %v", c.ladder["A"]) + t.Cleanup(func() { ompBenchJSON = prev }) + + c, out := scaffoldTo(t, "--bench") + for _, k := range c.keys { + if c.models[k].ID == "claude-mythos-5" { + t.Error("a model whose bench probe failed must not reach the ladder") + } } - if c.models[c.ladder["O"][3]].ID != "gpt-5.6-sol" { - t.Errorf("unexpected O ladder: %v", c.ladder["O"]) + body, _ := os.ReadFile(out) + if strings.Contains(string(body), "placeholder") { + t.Error("--bench should replace the placeholder speed/ttft, not annotate them") } - // A scaffolded catalog must render end-to-end. - if !strings.Contains(c.renderCatalog(), "\nmixed_smart_medium_nosp_nofa ") { - t.Error("scaffolded catalog fails to render") + if !strings.Contains(string(body), "speed: 48.9") || !strings.Contains(string(body), "ttft: 1.4") { + t.Errorf("measured figures missing from scaffold:\n%s", body) } - // Refuses to clobber an existing file. - if code := runGenerateInit([]string{"--from-json", src, "--models-file", out}); code == 0 { +} + +func TestGenerateInitRefresh(t *testing.T) { + stubOmp(t, initUsage) + dir := t.TempDir() + src := filepath.Join(dir, "omp.json") + out := filepath.Join(dir, "models.yml") + if err := os.WriteFile(src, []byte(initJSON), 0o644); err != nil { + t.Fatal(err) + } + base := []string{"--from-json", src, "--models-file", out} + if code := runGenerateInit(base); code != 0 { + t.Fatalf("first init exit %d", code) + } + // Without --refresh an existing file is left alone, so a stale catalog can + // never be clobbered by accident. + if code := runGenerateInit(base); code == 0 { t.Error("init must refuse to overwrite an existing models file") } + if err := os.WriteFile(out, []byte("models: {}\n"), 0o644); err != nil { + t.Fatal(err) + } + if code := runGenerateInit(append(base, "--refresh")); code != 0 { + t.Fatalf("refresh exit %d", code) + } + c, err := loadCatalog(out) + if err != nil { + t.Fatalf("refreshed scaffold does not load: %v", err) + } + if c.models[c.ladder["A"][3]].ID != "claude-opus-5" { + t.Error("--refresh should re-derive the tiers from the current model list") + } } diff --git a/onboarding.go b/onboarding.go index 1bc3ffe..ba01a97 100644 --- a/onboarding.go +++ b/onboarding.go @@ -74,7 +74,7 @@ func obScan() tea.Msg { if err != nil { return obScanDoneMsg{err: fmt.Errorf("running `omp models --json`: %w", err)} } - yml, err := scaffoldModels(raw) + yml, err := scaffoldModels(raw, nil) return obScanDoneMsg{scaffold: yml, err: err} } @@ -166,6 +166,11 @@ func (o onboarding) Update(msg tea.Msg) (tea.Model, tea.Cmd) { o.m.generated = blocks o.m.advisors = parseAdvisors(blocks["__advisors__"]) o.m.facts = parseFacts(blocks["__models__"]) + // A freshly scaffolded catalog may have no tier-0 or tier-4 model, in + // which case it ships no spark/fable combos at all — drop those dials + // before the real TUI paints, or the default selection lands on a + // combination that was never generated. + o.m.applyCatalog() o.m.syncPreview() return o.m, o.m.Init() case tea.KeyMsg: @@ -232,7 +237,7 @@ func (o onboarding) View() string { case obScanning: body = []string{o.spin.View() + " reading your omp model list…"} case obReview: - verb := "guessed from price — sanity-check it" + verb := "derived from your model list — sanity-check it" if o.existing { verb = "from your models file" } diff --git a/onboarding_test.go b/onboarding_test.go index 9d237d7..1d50ec3 100644 --- a/onboarding_test.go +++ b/onboarding_test.go @@ -28,6 +28,7 @@ func enter() tea.KeyMsg { return tea.KeyMsg{Type: tea.KeyEnter} } func TestOnboardingScanFlow(t *testing.T) { t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "cfg")) t.Setenv("XDG_DATA_HOME", filepath.Join(t.TempDir(), "data")) + stubOmp(t, initUsage) orig := ompModelsJSON ompModelsJSON = func() ([]byte, error) { return []byte(initJSON), nil } defer func() { ompModelsJSON = orig }() @@ -52,7 +53,7 @@ func TestOnboardingScanFlow(t *testing.T) { t.Fatalf("after scan: step=%v cat=%v", o.step, o.cat) } view := o.View() - for _, want := range []string{"claude-opus-4-8", "gpt-5.6-sol", "sanity-check"} { + for _, want := range []string{"claude-opus-5", "gpt-5.6-sol", "sanity-check"} { if !strings.Contains(view, want) { t.Errorf("review view missing %q", want) } From 8a694225fe3384c19187abcebb485954637290fc Mon Sep 17 00:00:00 2001 From: Alex TYRODE Date: Sat, 25 Jul 2026 02:23:29 +0000 Subject: [PATCH 2/5] feat: route scout and vision, and never launch a missing profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit omp bundles six agents; the grid routed five. `scout` — the one its own prompt marks "MUST be used for exploratory codebase research" — silently inherited @smol and never appeared in the preview, so its model was neither visible nor selectable. It is now an agent-backed role, which means genConfigYAML mirrors it into task.agentModelOverrides for free. `vision` backs omp's image-describe fallback (describeForTextModels is on by default) and always leads on a model that accepts images, since the codex spark variants are text-only. Both needed roleWeight entries: weightedModels silently skips a role it cannot weigh, so an omission drops that model out of the cost and speed meters with no trace. A test now asserts every emitted role is weighted. Enter no longer launches a combo the catalog does not carry. It walked a nil block and emitted an overlay whose modelRoles map was empty, handing omp a session with no routing at all — reproducible today on a stock scaffold, where the shipped default (spark on) points at a combo no tier-0-less catalog contains. The dials themselves are now derived from the catalog: a toggle with no combos is forced off and hidden rather than offered. The quota bucket a model draws is read from the catalog's new column, falling back to the family-substring guess only when a catalog declares none. Names are not a taxonomy — claude-mythos-5 sits at claude-fable-5's price yet 404s on this account, and every model omp adds would otherwise need one more substring arm before it could be struck through correctly. --- main.go | 124 ++++++++++++++++++++++++++++++++++++++----- main_test.go | 146 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 250 insertions(+), 20 deletions(-) diff --git a/main.go b/main.go index f4ecf6d..a050fe4 100644 --- a/main.go +++ b/main.go @@ -181,7 +181,10 @@ func paintModel(tok string) string { func colorizeRoute(line string) string { return modelRe.ReplaceAllStringFunc(line, paintModel) } -// providerOf maps a short/long model name to its quota bucket. +// bucketOf guesses a quota bucket from a model name. It is the fallback for +// catalogs that declare no bucket column, and the only resolver for the bare +// facet names ("fable", "spark") the suggest box asks about — prefer +// model.bucketFor wherever a receiver is in reach. func bucketOf(model string) string { m := model if i := strings.IndexByte(m, ':'); i >= 0 { @@ -199,6 +202,23 @@ func bucketOf(model string) string { return "codex-main" } +// bucketFor resolves a routing token's quota bucket from the catalog, falling +// back to the name guess only when the catalog declares none. The catalog wins +// because names are not a taxonomy: claude-mythos-5 sits in omp's catalog at +// claude-fable-5's price yet 404s on this account, and every model omp adds +// would otherwise need one more substring arm here before it could be struck +// through correctly. +func (m model) bucketFor(name string) string { + id := name + if i := strings.IndexByte(id, ':'); i >= 0 { + id = id[:i] + } + if f, ok := m.facts[id]; ok && f.bucket != "" { + return f.bucket + } + return bucketOf(id) +} + // ── data ───────────────────────────────────────────────────────────────────── // loadBlocks parses a generated page into name -> role rows. func loadBlocks(path string) map[string][]string { @@ -852,8 +872,9 @@ func selectedAvailability(a availability, disabled map[accountKey]bool) availabi // renderRoute lays out each role's chain, wrapping cleanly at `width`: when a // chain doesn't fit, it breaks after an arrow and the continuation is indented // to align under the first model, so it reads as one hanging block rather than a -// ragged wrap. Down (maxed/unauthed) models are struck through. -func renderRoute(rows []string, depth int, a availability, width int) string { +// ragged wrap. Down (maxed/unauthed) models are struck through — which bucket a +// model draws from is a catalog fact, hence the receiver. +func (m model) renderRoute(rows []string, depth int, a availability, width int) string { if width < 24 { width = 24 } @@ -876,7 +897,7 @@ func renderRoute(rows []string, depth int, a availability, width int) string { var toks []tok for _, loc := range locs { mt := r[loc[0]:loc[1]] - toks = append(toks, tok{mt, a.ok && a.down(bucketOf(mt))}) + toks = append(toks, tok{mt, a.ok && a.down(m.bucketFor(mt))}) } // how many to show: full → all; lead → primary, or up to the first live // model when the lead is down (the one that actually runs). @@ -984,9 +1005,11 @@ func parseAdvisors(rows []string) map[string][]string { } // modelFact is a model's measured facts from omp (via the catalog): pricing -// ($/1M tokens), output throughput (tok/s), and time-to-first-token (seconds). +// ($/1M tokens), output throughput (tok/s), time-to-first-token (seconds), and +// the quota bucket it draws from ("" when the catalog declares none). type modelFact struct { in, out, speed, ttft float64 + bucket string } // effTPS folds ttft into throughput — the effective tok/s for a representative @@ -1001,8 +1024,10 @@ func (f modelFact) effTPS() float64 { return effTokens / (f.ttft + effTokens/f.speed) } -// parseFacts reads the __models__ block (rows: " ") -// into a per-model table, sourced from the catalog so meters and routing agree. +// parseFacts reads the __models__ block (rows: " +// []") into a per-model table, sourced from the catalog so meters and +// routing agree. The bucket column is optional — a catalog that declares none +// for any model omits it entirely, and the name guess covers those rows. func parseFacts(rows []string) map[string]modelFact { out := map[string]modelFact{} for _, r := range rows { @@ -1014,8 +1039,12 @@ func parseFacts(rows []string) map[string]modelFact { outc, e2 := strconv.ParseFloat(f[2], 64) sp, e3 := strconv.ParseFloat(f[3], 64) tt, e4 := strconv.ParseFloat(f[4], 64) + bucket := "" + if len(f) >= 6 { + bucket = f[5] + } if e1 == nil && e2 == nil && e3 == nil && e4 == nil { - out[f[0]] = modelFact{in, outc, sp, tt} + out[f[0]] = modelFact{in, outc, sp, tt, bucket} } } return out @@ -1031,10 +1060,12 @@ func parseFacts(rows []string) map[string]modelFact { // scale with thinking effort (more reasoning = pricier + slower) and take OpenAI's // priority tier under fast mode (pricier but quicker). The weighted averages map // onto 1..5 log scales (both perceived multiplicatively), calibrated across every -// valid facet × advisor × fast combination. +// valid facet × advisor × fast combination. Every role the generator emits must +// appear here: weightedModels silently skips a role it cannot weigh, so an +// omission drops that model out of both meters with no trace. var roleWeight = map[string]float64{ - "default": 10, "task": 6, "reviewer": 3, "sonic": 3, "plan": 3, "advisor": 4, - "slow": 2, "designer": 2, "librarian": 2, "smol": 1, "tiny": 0.5, "commit": 0.5, + "default": 10, "task": 6, "reviewer": 3, "sonic": 3, "plan": 3, "advisor": 4, "slow": 2, + "designer": 2, "librarian": 2, "scout": 2, "smol": 1, "tiny": 0.5, "commit": 0.5, "vision": 0.5, } var thinkMult = map[string]float64{ // reasoning tokens grow with effort → pricier "minimal": 0.6, "low": 0.8, "medium": 1.0, "high": 1.3, "xhigh": 1.6, "max": 2.0, @@ -1218,7 +1249,8 @@ func (m model) applyAdvisor(rows []string, level string) []string { // visibleFacets drops facets that don't apply to the current lane, so the // generator only ever shows actionable options: no spark/fast on a Claude-only // pool, no fable on a GPT-only pool. main is fable's sub-setting, so it only -// shows while fable is on (and the lane can host it at all). +// shows while fable is on (and the lane can host it at all). A dial this catalog +// generated no combo for is dropped the same way — it is not a choice. func (m model) visibleFacets() []facet { lane := m.sel["lane"] var out []facet @@ -1229,6 +1261,12 @@ func (m model) visibleFacets() []facet { if lane == "gpt-only" && (f.key == "fable" || f.key == "main") { continue } + if f.key == "spark" && m.noSpark { + continue + } + if (f.key == "fable" || f.key == "main") && m.noFable { + continue + } if f.key == "main" && m.sel["fable"] != "on" { continue } @@ -1259,6 +1297,43 @@ func comboID(sel map[string]string) string { return fmt.Sprintf("%s_%s_%s_%s_%s", lane, sel["model"], sel["thinking"], spid, faid) } +// applyCatalog records which dials this catalog can actually serve, then forces +// the rest off. A models file with no tier-0 model yields no _sp_ combos at all, +// so the shipped default (spark on) would open the TUI on a combo that was never +// written — and a selection persisted against a richer catalog does the same. +// Ids are ____, so match whole +// segments: "nosp" and "nofa" contain the very substrings being looked for. +func (m *model) applyCatalog() { + if len(m.generated) == 0 { + return // no catalog read yet: onboarding, or a broken CODE_GENERATED + } + spark, fable := false, false + for id := range m.generated { + for _, seg := range strings.Split(id, "_") { + switch seg { + case "sp": + spark = true + case "fa", "famain": + fable = true + } + } + } + m.noSpark, m.noFable = !spark, !fable + m.clampSel() +} + +// clampSel turns off every dial the catalog cannot serve. main is fable's +// sub-setting and never outlives it. +func (m *model) clampSel() { + if m.noSpark { + m.sel["spark"] = "off" + } + if m.noFable { + m.sel["fable"] = "off" + m.sel["main"] = "off" + } +} + func laneColor(lane string) string { switch lane { case "gpt-only": @@ -1689,6 +1764,12 @@ type model struct { avail availability glyphs map[string]string + // Catalog capability, phrased as absence so the zero value keeps every dial: + // a model with no catalog yet (the onboarding shell, tests) must behave as it + // always did. applyCatalog sets these only from a catalog it actually read. + noSpark bool // no _sp_ combos exist — hide the spark dial and force it off + noFable bool // no _fa_/_famain_ combos — same for fable and its main child + depth int // 0 lead · 1 full collapse bool // p: hide the Routing section showResult bool // in collapsed mode: show the preview full-width @@ -2619,7 +2700,7 @@ func (m *model) syncPreviewAt(yoff int) { if base, ok := m.generated[id]; ok { _, roles := splitMeta(base) roles = m.applyAdvisor(roles, m.sel["advisor"]) - b.WriteString(renderRoute(roles, m.depth, m.selectedLaunchAvailability(), rw)) + b.WriteString(m.renderRoute(roles, m.depth, m.selectedLaunchAvailability(), rw)) } else { b.WriteString(stDim.Render("no profile for this combination") + "\n") } @@ -2895,6 +2976,7 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.relayout() // the taller/shorter footer changes the body height case "d": m.sel = defaultSel() + m.clampSel() // the defaults assume a full catalog; this one may not be m.persistSelection() m.syncPreview() case "f": @@ -2938,6 +3020,14 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case "enter": // Enter always launches the generated profile for the current facets — // the untouched default combo is a generated profile like any other. + // Never for a combo the catalog doesn't carry, though: genConfigYAML + // would walk a nil block and emit an overlay whose modelRoles map is + // empty, handing omp a session with no routing at all. The preview + // already says "no profile for this combination", so the key does + // nothing rather than launching something broken. + if _, ok := m.generated[comboID(m.sel)]; !ok { + return m, nil + } m.genConfig = m.genConfigYAML() return m, tea.Quit } @@ -2953,6 +3043,11 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { m.savedSel[k] = v } m.applyActions(msg.Actions) + // applyActions' repair rules know lanes and quota, not catalog contents: + // a "critical" proposal switches fable on even where no fable combo was + // generated. Clamp and re-render before reporting what was applied. + m.clampSel() + m.syncPreview() return m, func() tea.Msg { return clikit.AppliedActionsMsg{Actions: m.appliedDiff()} } case clikit.ActionsConfirmedMsg: @@ -3356,6 +3451,9 @@ func main() { selectionState: selectionState, hasSandbox: hasSandbox, } + // The catalog decides which dials exist at all; a persisted or default + // selection must not open on a combo it never generated. + m.applyCatalog() // First run: no catalog anywhere and no explicit CODE_GENERATED — wrap the // TUI in the guided onboarding that builds one (an explicit but broken // CODE_GENERATED is an operator config error and is left visible as the diff --git a/main_test.go b/main_test.go index cd425e3..afba52e 100644 --- a/main_test.go +++ b/main_test.go @@ -55,7 +55,7 @@ func TestRenderRouteHangingIndent(t *testing.T) { indent := strings.Repeat(" ", lw) // The sample chain is ~70 cols on one line; these widths force it to wrap. for _, width := range []int{40, 52, 64} { - out := renderRoute([]string{sampleRow}, 1, availability{}, width) + out := model{}.renderRoute([]string{sampleRow}, 1, availability{}, width) lines := routeLines(out) if len(lines) < 2 { t.Fatalf("width=%d: expected the chain to wrap onto multiple lines, got %d", width, len(lines)) @@ -77,7 +77,7 @@ func TestRenderRouteHangingIndent(t *testing.T) { // 2-col reserve for the trailing arrow must keep even break lines in bounds. func TestRenderRouteWidthInvariant(t *testing.T) { for _, width := range []int{40, 56, 72, 100} { - out := renderRoute([]string{sampleRow}, 1, availability{}, width) + out := model{}.renderRoute([]string{sampleRow}, 1, availability{}, width) for i, ln := range routeLines(out) { if w := lipgloss.Width(ln); w > width { t.Errorf("width=%d: line %d overflows (%d cols): %q", width, i, w, ln) @@ -88,7 +88,7 @@ func TestRenderRouteWidthInvariant(t *testing.T) { // TestRenderRouteLeadDepth: at depth 0 only the primary (first live) model shows. func TestRenderRouteLeadDepth(t *testing.T) { - out := renderRoute([]string{sampleRow}, 0, availability{}, 120) + out := model{}.renderRoute([]string{sampleRow}, 0, availability{}, 120) if lines := routeLines(out); len(lines) != 1 { t.Fatalf("lead depth should be a single line, got %d:\n%s", len(lines), out) } @@ -103,19 +103,75 @@ func TestRenderRouteLeadDepth(t *testing.T) { // TestRenderRoutePassThrough: a line with no models is emitted unchanged (modulo // colourisation), not dropped. func TestRenderRoutePassThrough(t *testing.T) { - out := renderRoute([]string{" advisor (disabled)"}, 1, availability{}, 80) + out := model{}.renderRoute([]string{" advisor (disabled)"}, 1, availability{}, 80) if !strings.Contains(out, "advisor") || !strings.Contains(out, "(disabled)") { t.Errorf("note line should pass through, got: %q", out) } } +// TestParseFactsBucketColumn: the __models__ bucket column is optional — a +// catalog that declares no bucket for any model omits it entirely — so both row +// widths must parse, while anything short of the five numeric facts is dropped. +func TestParseFactsBucketColumn(t *testing.T) { + facts := parseFacts([]string{ + " gpt-5.6-luna 1 6 52.3 1.18 codex-main", + " gpt-5.6-terra 2.5 15 41 1.4", + " claude-mythos-5 5 25 30 2 claude-fable", + " truncated 1 2 3", + " unparseable 1 2 3 later codex-main", + }) + if len(facts) != 3 { + t.Fatalf("parsed %d rows, want 3: %v", len(facts), facts) + } + want := map[string]modelFact{ + "gpt-5.6-luna": {1, 6, 52.3, 1.18, "codex-main"}, + "gpt-5.6-terra": {2.5, 15, 41, 1.4, ""}, + "claude-mythos-5": {5, 25, 30, 2, "claude-fable"}, + } + for id, w := range want { + if got := facts[id]; got != w { + t.Errorf("parseFacts[%q] = %+v, want %+v", id, got, w) + } + } +} + +// TestBucketForPrefersCatalog: the catalog's bucket column beats the name guess. +// claude-mythos-5 prices like fable and drains fable's quota, but every +// substring arm in bucketOf reads it as plain claude-main — the routing preview +// would then strike models through against the wrong window. +func TestBucketForPrefersCatalog(t *testing.T) { + m := model{facts: map[string]modelFact{ + "claude-mythos-5": {5, 25, 30, 2, "claude-fable"}, + "gpt-5.6-luna": {1, 6, 52.3, 1.18, ""}, + }} + if got := bucketOf("claude-mythos-5"); got != "claude-main" { + t.Fatalf("guess baseline moved: bucketOf(claude-mythos-5) = %q", got) + } + if got := m.bucketFor("claude-mythos-5:high"); got != "claude-fable" { + t.Errorf("catalog bucket ignored: bucketFor(claude-mythos-5:high) = %q", got) + } + // An undeclared bucket, and a model the catalog never mentions: both guess. + if got := m.bucketFor("gpt-5.6-luna:low"); got != "codex-main" { + t.Errorf("empty bucket must fall back to the guess, got %q", got) + } + if got := m.bucketFor("claude-fable-5:max"); got != "claude-fable" { + t.Errorf("unknown model must fall back to the guess, got %q", got) + } + // The suggest box asks about bare facet names, which are never catalog ids. + for name, want := range map[string]string{"fable": "claude-fable", "spark": "codex-spark"} { + if got := m.bucketFor(name); got != want { + t.Errorf("bucketFor(%q) = %q, want %q", name, got, want) + } + } +} + func TestShortModel(t *testing.T) { cases := map[string]string{ "gpt-5.6-terra": "terra", "gpt-5.6-luna": "luna", "gpt-5.6-sol": "sol", "gpt-5.3-codex-spark": "spark", - "claude-opus-4-8": "opus", + "claude-opus-5": "opus", "claude-sonnet-5": "sonnet", "claude-haiku-4-5": "haiku", "claude-fable-5": "fable", @@ -221,6 +277,68 @@ func TestMainFacetVisibility(t *testing.T) { } } +// TestCatalogCapabilityGatesDials: spark and fable must never point at a combo +// the catalog cannot serve. A models file with no tier-0 model generates zero +// _sp_ ids, so the shipped default (spark on) would open the TUI on a combo that +// was never written. The segment check must not be fooled by the "nosp"/"nofa" +// ids that spell the very substrings it looks for. +func TestCatalogCapabilityGatesDials(t *testing.T) { + visible := func(m model) map[string]bool { + out := map[string]bool{} + for _, f := range m.visibleFacets() { + out[f.key] = true + } + return out + } + + full := model{facets: facetDefs(map[string]string{}), sel: defaultSel(), + generated: map[string][]string{ + "mixed_smart_medium_sp_fa": nil, + "mixed_smart_medium_nosp_nofa": nil, + }} + full.applyCatalog() + if full.noSpark || full.noFable { + t.Fatalf("a catalog serving both dials disabled one: noSpark=%v noFable=%v", full.noSpark, full.noFable) + } + if full.sel["spark"] != "on" { + t.Errorf("spark must survive a catalog that serves it, got %q", full.sel["spark"]) + } + + bare := model{facets: facetDefs(map[string]string{}), sel: defaultSel(), + generated: map[string][]string{ + "mixed_smart_medium_nosp_nofa": nil, + "gpt-only_fast_low_nosp_nofa": nil, + "__models__": nil, + }} + bare.sel["fable"], bare.sel["main"] = "on", "on" + bare.applyCatalog() + if !bare.noSpark || !bare.noFable { + t.Fatalf(`"nosp"/"nofa" ids read as capability: noSpark=%v noFable=%v`, bare.noSpark, bare.noFable) + } + for key, want := range map[string]string{"spark": "off", "fable": "off", "main": "off"} { + if got := bare.sel[key]; got != want { + t.Errorf("%s stayed %q on a catalog that cannot serve it, want %q", key, got, want) + } + } + if v := visible(bare); v["spark"] || v["fable"] || v["main"] { + t.Errorf("unusable dials stayed on screen: %v", v) + } + + // The reset key restores defaults, which assume a full catalog. + reset, _ := bare.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune{'d'}}) + if got := reset.(model).sel["spark"]; got != "off" { + t.Errorf("d reset resurrected the dead spark dial: %q", got) + } + + // No catalog at all is the onboarding shell, not a restriction. + empty := model{facets: facetDefs(map[string]string{}), sel: defaultSel()} + empty.applyCatalog() + if empty.noSpark || empty.noFable || empty.sel["spark"] != "on" { + t.Errorf("an unread catalog must leave every dial alone: noSpark=%v noFable=%v spark=%q", + empty.noSpark, empty.noFable, empty.sel["spark"]) + } +} + // TestCycleFacetClearsMain: manually toggling fable off must clear fable-as-main // too, so a later fable re-enable never silently resurrects the escalation. func TestCycleFacetClearsMain(t *testing.T) { @@ -285,6 +403,20 @@ func TestLaunchKeys(t *testing.T) { } } +// TestEnterRefusesMissingCombo: Enter must not launch facets the catalog carries +// no block for. genConfigYAML walks a nil block and emits an overlay whose +// modelRoles map is empty, which would hand omp a session with no routing at +// all — while the preview says "no profile for this combination". +func TestEnterRefusesMissingCombo(t *testing.T) { + m := model{sel: defaultSel(), generated: map[string][]string{}} + next, cmd := m.Update(tea.KeyMsg{Type: tea.KeyEnter}) + got := next.(model) + if got.genConfig != "" || cmd != nil { + t.Fatalf("Enter launched a combo with no generated block: genConfig=%q quit=%v", + got.genConfig, cmd != nil) + } +} + // TestGenConfigYAMLAgentOverrides locks the atyrode/dotfiles#173 fix: every ●-marked // agent-backed role in the generated block is mirrored into // task.agentModelOverrides (so spawned agents follow the generated profile), @@ -494,7 +626,7 @@ func layoutModel() model { " default gpt-5.6-terra:medium → gpt-5.6-luna:medium → claude-sonnet-5:medium", " ● task gpt-5.6-terra:medium → gpt-5.6-luna:medium → claude-sonnet-5:medium", " ● scout gpt-5.6-luna:low → claude-haiku-4-5:low", - " advisor claude-opus-4-5:high", + " advisor claude-opus-5:high", " commit gpt-5.6-luna:minimal", } return model{ @@ -1823,7 +1955,7 @@ func TestRoutingFallbackCuePinned(t *testing.T) { lines := strings.Split(stripAnsi(m.View()), "\n") cue := lineIndex(lines, "f · show fallback chains") title := lineIndex(lines, "routing", "p · hide") - route := lineIndex(lines, "scout") // a routing role row — generator has none + route := lineIndex(lines, "scout") // an agent-backed routing role row if cue < 0 || title < 0 || route < 0 { t.Fatalf("%s: missing cue (%d), title (%d), or route content (%d):\n%s", label, cue, title, route, strings.Join(lines, "\n")) From 12fc6cd449d96dd8d1a5c48d1bb05bb4c9345e76 Mon Sep 17 00:00:00 2001 From: Alex TYRODE Date: Sat, 25 Jul 2026 02:23:44 +0000 Subject: [PATCH 3/5] docs: correct the env table, the first-run story, and two known gaps - docs/configuration.md documented CODE_USAGE as the usage panel's source; nothing reads it, and the panel has been broker-sourced since the account-pool work. Removed, with a note that the wrapper still exports it for older pins so a reader who greps it is not misled. Seven vars the binary does read were undocumented: the three OMP_AUTH_BROKER_*, CODE_AUTH_VAULTS(_FILE), CODE_AUTH_ACCOUNT_STATE and CODE_USAGE_CACHE. - README claimed the first run walks you through building a catalog. True for a plain install; the dotfiles wrapper always exports CODE_GENERATED, so onboarding cannot trigger there. - docs/status.md's two honest gaps are narrowed rather than dropped: speed/ttft placeholders now have --bench, and the quota bucket is a catalog fact with the family guess as fallback. - Documents the generate subcommand, which the README linked to but the page never covered. --- README.md | 4 +++ docs/configuration.md | 59 ++++++++++++++++++++++++++++++++++++++++++- docs/status.md | 32 +++++++++++++++-------- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 582dc73..2be816f 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,10 @@ using `omp auth-broker login` before running `code`. Then just run `code`. The first run notices there's no routing catalog yet and walks you through building one from your omp's model list — it shows you which model it picked for each rung, you sanity-check, press enter, done. +That guided run is for plain installs: the +[dotfiles](https://github.com/atyrode/dotfiles) wrapper always exports +`CODE_GENERATED` at a pre-baked catalog, so `code` never offers to build one +there — you re-render with `code generate` instead. The same machinery is scriptable as `code generate init` (scaffold the models file) and `code generate` (re-render the catalog after you edit it). diff --git a/docs/configuration.md b/docs/configuration.md index a7cd5a9..d06a356 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -28,18 +28,75 @@ environment variable with a sane fallback. | Variable | Purpose | Without it | |---|---|---| | `CODE_GENERATED` | path to the generated facet catalog (the routing blocks behind the dials) | `$XDG_DATA_HOME/code/generated.plain`, where `code generate` writes; if that's missing too, the TUI opens the guided first-run that builds it | -| `CODE_USAGE` | command printing `omp usage --json` for the usage panel | panel hidden | | `CODE_SELECTION_STATE` | file persisting your dial choices | choices reset each run | | `CODE_SESSION_STATE` | directory recording live sessions for `code ls` / `code session reap`; `off` disables recording | `$XDG_STATE_HOME/code/sessions` — note this one defaults to a path rather than to disabled, so the registry works without wrapper changes | | `CODE_OMP` | omp binary for trusted launches (`m` and `enter`) | `omp-managed`, then `omp` on PATH | | `CODE_OMP_UNTRUSTED` | sandboxed omp for the `u` key | `ompu` on PATH, else the key is hidden and inert | +| `OMP_AUTH_BROKER_URL` | central auth broker behind the usage panel and the account picker (`v`); inherited from your omp environment | no fetch — the usage panel has nothing to show | +| `OMP_AUTH_BROKER_TOKEN` | bearer token for that broker | same: `code` only fetches when both the URL and the token are set | +| `OMP_AUTH_BROKER_SNAPSHOT_CACHE` | broker snapshot cache path; `code` never reads it, it only forwards it to the omp it launches | forwarded empty | +| `CODE_AUTH_VAULTS` | legacy vault manifest (inline JSON), consulted only when no `OMP_AUTH_BROKER_*` variable is set | the broker variables are the only source | +| `CODE_AUTH_VAULTS_FILE` | the same legacy manifest read from a file, when `CODE_AUTH_VAULTS` is empty | ditto | +| `CODE_AUTH_ACCOUNT_STATE` | file persisting your broker account selections and presets (`v`) | selections reset each run | +| `CODE_USAGE_CACHE` | file caching the last usage snapshot, so the panel opens on last-known numbers (marked stale) instead of blank | the panel starts empty and fills on the first fetch | | `CODE_EVAL_MODEL` | ollama model tag for `ctrl+o` | `qwen2.5:3b` | | `CODE_OLLAMA_ENDPOINT` | non-default ollama endpoint | `http://127.0.0.1:11434` | | `CODE_FACET_GLYPHS` | override the Nerd Font dial glyphs | built-in glyphs | +`CODE_USAGE` and `CODE_OMP_RAW` are no longer read; the dotfiles wrapper still +exports them for older pinned builds. The usage panel now comes from the auth +broker (`OMP_AUTH_BROKER_URL` / `OMP_AUTH_BROKER_TOKEN`). + Provider authentication is owned by OMP, not `code`. Authenticate with `omp auth-broker login` before launching `code`. +## The `code generate` subcommand + +The dials are backed by a pre-rendered catalog. Building it is two steps, both +scriptable: + +``` +code generate init [--from-json FILE] [--models-file OUT] [--bench] [--refresh] +code generate [--models-file FILE] [--out FILE|-] +``` + +`init` scaffolds a models file from your own omp (`omp models --json`, or +`FILE`), keeping the newest model per family and ranking it by thinking +ceiling, context and price — review what it derived. It also reads +`omp usage --json`: a quota bucket scoped to a model tier is how the spark and +elite rungs are identified, and without that report they are simply left empty. +`generate` +renders that file into the catalog the TUI reads. Paths default to +`$XDG_CONFIG_HOME/code/models.yml` and `$XDG_DATA_HOME/code/generated.plain` +(`~/.config` and `~/.local/share` when those are unset); `--out -` prints the +catalog to stdout. + +| Flag | Effect | +|---|---| +| `--bench` | measure `speed` and `ttft` per model with `omp bench --json` instead of writing placeholders. It doubles as a reachability probe: omp lists models your account cannot actually call, and any model whose probe fails is dropped from the ladder — nothing else catches that class of error. Slow (one real API call per model) and needs live credentials | +| `--refresh` | re-derive the tiers over an existing models file instead of refusing to touch it. Without it `init` stops when the file already exists, so a scaffold from months ago keeps naming retired models. Pair the two — `code generate init --refresh --bench` — when a provider ships new models; that is the exact line the scaffolder leaves in its own placeholder comment | + +### The models file + +`models:` maps a short key to one model: + +| Field | Meaning | +|---|---| +| `id` | the model id omp routes to | +| `pool` | `O` (OpenAI/Codex) or `A` (Anthropic) | +| `tier` | `1` cheap · `2` regular · `3` smart — the per-pool fallback ladder. `0` (a fast idle-bucket model the `spark` toggle drains) and `4` (a scarce elite the `fable` toggle leads with) are optional | +| `bucket` | the quota window this model draws from (`claude-main`, `claude-fable`, `codex-main`, `codex-spark`). The TUI prefers it over guessing from the model family | +| `cost_in` / `cost_out` | dollars per 1M tokens; drives the cost meter | +| `speed` / `ttft` | output tok/s and seconds to first token; drives the speed meter. Placeholders unless you ran `--bench` or filled them in yourself | +| `context` | context window, in tokens | +| `thinking` | the levels the model really offers (see below) | +| `image` | omitted for image-capable models, which is most of them. `init` writes `image: false` only for a model omp reports as text-only, and the `vision` role then avoids it | + +The thinking scale is `minimal · low · medium · high · xhigh · max`. Write +`low→max` for a contiguous run, or a comma list when the model skips a level: +claude-opus-4-6 offers `low,medium,high,max` but not `xhigh`, and a range there +would claim a level the API rejects. + ## The `ctrl+o` classifier Any ollama daemon on loopback works: diff --git a/docs/status.md b/docs/status.md index 0a7bf7c..e3396b2 100644 --- a/docs/status.md +++ b/docs/status.md @@ -13,20 +13,32 @@ provider with defaults, you probably don't need it. ## The catalog The dials map to pre-generated routing blocks. `code generate init` scaffolds -a models file from your own omp instance (`omp models --json`) and -`code generate` renders the catalog from it — see the README quickstart. Two -honest limits: the tier assignments `init` guesses from price deserve a human -look, and the speed/ttft numbers it writes are placeholders (they only drive -the TUI's speed meter) until you measure and update them. +a models file from your own omp instance (`omp models --json`, plus +`omp usage --json` to spot the tier-scoped quota buckets that mark the spark +and elite models) and `code generate` renders the catalog from it — see the +README quickstart. Two +honest limits: the tier assignments `init` derives (newest model per family, +then ranked by thinking ceiling, context and price) deserve a human look, and +the speed/ttft numbers it writes are placeholders unless you ask for +measured ones. `code generate init --bench` fills both from `omp bench --json` +— time to first token and output tok/s, one real API call per model, live +credentials required — and drops any model whose probe fails, which is the only +check that catches a model omp lists but your account cannot actually call. +Without it every model carries the identical placeholder pair, so the speed +meter those numbers drive is model-invariant: it moves with the thinking dial +and nothing else. ## Other honest caveats -- oh-my-pi releases near-daily, and the `omp models --json` / - `omp usage --json` schemas the usage panel and (future) generator rely on - carry no stability guarantee. A scheduled compatibility check is planned: +- oh-my-pi releases near-daily, and the `omp models --json` / `omp usage --json` + (and, with `--bench`, `omp bench --json`) schemas the generator reads carry no + stability guarantee — nor does the auth broker's snapshot/usage API the + panel draws from. A scheduled compatibility check is planned: [#3](https://github.com/atyrode/code/issues/3). -- Some quota heuristics (bucket names, model-family colouring) reflect the - author's provider mix. They fail soft. +- The quota bucket a model draws from is declared in the catalog (`bucket:`) + and the TUI prefers that; guessing it from the model family is now only the + fallback for catalogs that declare none. Model-family colouring is still + name matching that reflects the author's provider mix. Both fail soft. ## Built on From 80afe908c029c8ee44fe9ee77bbe0c91b9d923e5 Mon Sep 17 00:00:00 2001 From: Alex TYRODE Date: Sat, 25 Jul 2026 04:02:36 +0000 Subject: [PATCH 4/5] feat(generate): verify every rung is callable before it can route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit omp lists models an account cannot call, and nothing in the metadata says which. `claude-mythos-5` reports `claude-fable-5`'s exact price, context window and thinking range, and 404s here. Until now the only thing keeping it off the ladder was a shorter-id tiebreak — rename it `claude-myth-5` and it wins the smart rung on this account, and every profile leading with it fails at launch. So reachability is no longer a flag. `init` probes every candidate and only a model that answers may become a rung; the file is stamped `probed: true`, and `code generate` refuses one that is not. That marker is the whole gate: `--from-json` still scaffolds offline, but stamps `false`, so an unverified scaffold cannot quietly become live routing. `--bench` is gone — its measurement is now a side effect of a probe that always runs, which also retires the placeholder speed/ttft pair that made the speed meter identical for every model. A failed request is not the same as an absent model, and conflating them is expensive. Probing the real catalog, `claude-fable-5` failed: omp's bundled bench prompt trips Anthropic's safety layer, and the model refuses it. A rule of "any failure means unreachable" would have silently deleted a perfectly callable elite from the ladder. The probe therefore asks something innocuous, and sorts each model three ways: reachable answered — eligible, and measured not found the provider says it does not exist — dropped unresolved refusal, rate limit, missing row — nothing is inferred Unresolved is a refusal to certify, not a quiet drop: `init` names each model and its reason and writes nothing, because a `probed: true` file built from a partial probe is worse than no file. Second defect, same shape: the elite tier was read only from omp's tier-scoped quota buckets, and that report turns out to depend on the ambient auth environment — the anthropic fable bucket is visible from one environment on this machine and absent from another, minutes apart. When it was absent the elite did not merely go undetected, it fell back onto the ordinary ladder and was crowned tier 3, so every routine "smart" request would drain the scarce bucket that exists to be spent deliberately. Price now confirms what the bucket used to assert alone: a pool's top model priced at twice the next is its elite, bucket or no bucket. Price says nothing about entitlement — that is the probe's job — but a model in its own price class is not the everyday workhorse. Verified against the live catalog: 21 models probed in ~47s, claude-mythos-5 dropped, claude-fable-5 kept and correctly crowned tier 4, yielding haiku/sonnet-5/opus-5/fable-5 and spark/mini/terra/sol. --- docs/configuration.md | 45 +++-- docs/status.md | 27 +-- generate.go | 32 +++- generate_init.go | 201 ++++++++++++++++++---- generate_test.go | 390 ++++++++++++++++++++++++++++++++++++++---- onboarding.go | 19 +- onboarding_test.go | 3 + 7 files changed, 612 insertions(+), 105 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d06a356..956ee9d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -56,29 +56,50 @@ The dials are backed by a pre-rendered catalog. Building it is two steps, both scriptable: ``` -code generate init [--from-json FILE] [--models-file OUT] [--bench] [--refresh] +code generate init [--models-file OUT] [--refresh] [--from-json FILE] code generate [--models-file FILE] [--out FILE|-] ``` -`init` scaffolds a models file from your own omp (`omp models --json`, or -`FILE`), keeping the newest model per family and ranking it by thinking -ceiling, context and price — review what it derived. It also reads -`omp usage --json`: a quota bucket scoped to a model tier is how the spark and -elite rungs are identified, and without that report they are simply left empty. -`generate` -renders that file into the catalog the TUI reads. Paths default to +`init` scaffolds a models file from your own omp (`omp models --json`), keeping +the newest model per family and ranking it by thinking ceiling, context and +price — review what it derived. It also reads `omp usage --json`: a quota bucket +scoped to a model tier is how the spark and elite rungs are identified, and +without that report they are simply left empty. `generate` renders that file +into the catalog the TUI reads. Paths default to `$XDG_CONFIG_HOME/code/models.yml` and `$XDG_DATA_HOME/code/generated.plain` (`~/.config` and `~/.local/share` when those are unset); `--out -` prints the catalog to stdout. +Every candidate is probed with `omp bench` before it can become a rung. This is +not optional and not a benchmark: omp lists models your account cannot actually +call, and no field distinguishes them — `claude-mythos-5` reports +`claude-fable-5`'s exact price, context window and thinking range, and 404s. +A model that does not return a passing probe is dropped, and a model missing +from the probe report entirely is dropped too, because unverified is not the +same as fine. One request per model, so expect `init` to take a minute. The +probe also supplies the real `speed`/`ttft` the meter reads. + +A file whose models were all verified is marked `probed: true`, and `generate` +refuses to render one that is not — that marker is the only thing standing +between an unverified scaffold and live routing. Treat it as your attestation +rather than a permanent certificate: it describes the ids as they were written, +so if you edit an `id` by hand, re-run `init --refresh` (or satisfy yourself the +new one is callable) instead of leaving the old `true` in place. + | Flag | Effect | |---|---| -| `--bench` | measure `speed` and `ttft` per model with `omp bench --json` instead of writing placeholders. It doubles as a reachability probe: omp lists models your account cannot actually call, and any model whose probe fails is dropped from the ladder — nothing else catches that class of error. Slow (one real API call per model) and needs live credentials | -| `--refresh` | re-derive the tiers over an existing models file instead of refusing to touch it. Without it `init` stops when the file already exists, so a scaffold from months ago keeps naming retired models. Pair the two — `code generate init --refresh --bench` — when a provider ships new models; that is the exact line the scaffolder leaves in its own placeholder comment | +| `--refresh` | re-derive the tiers over an existing models file instead of refusing to touch it. Without it `init` stops when the file already exists, so a scaffold from months ago keeps naming retired models. This is the line to run when a provider ships new models | +| `--from-json` | read the model list from a file instead of omp, and skip the probe. Offline inspection only: the output is marked `probed: false`, which `generate` rejects | ### The models file -`models:` maps a short key to one model: +Two top-level keys: `probed`, and `models:` mapping a short key to one model. + +| Top-level field | Meaning | +|---|---| +| `probed` | must be `true` or `generate` refuses the file. `init` sets it after every model passed a live probe; an offline `--from-json` scaffold writes `false` | + +Each entry under `models:`: | Field | Meaning | |---|---| @@ -87,7 +108,7 @@ catalog to stdout. | `tier` | `1` cheap · `2` regular · `3` smart — the per-pool fallback ladder. `0` (a fast idle-bucket model the `spark` toggle drains) and `4` (a scarce elite the `fable` toggle leads with) are optional | | `bucket` | the quota window this model draws from (`claude-main`, `claude-fable`, `codex-main`, `codex-spark`). The TUI prefers it over guessing from the model family | | `cost_in` / `cost_out` | dollars per 1M tokens; drives the cost meter | -| `speed` / `ttft` | output tok/s and seconds to first token; drives the speed meter. Placeholders unless you ran `--bench` or filled them in yourself | +| `speed` / `ttft` | output tok/s and seconds to first token; drives the speed meter. Measured by `init`'s probe — a single timed request each, so treat them as one sample rather than a stable benchmark | | `context` | context window, in tokens | | `thinking` | the levels the model really offers (see below) | | `image` | omitted for image-capable models, which is most of them. `init` writes `image: false` only for a model omp reports as text-only, and the `vision` role then avoids it | diff --git a/docs/status.md b/docs/status.md index e3396b2..d732f03 100644 --- a/docs/status.md +++ b/docs/status.md @@ -16,22 +16,25 @@ The dials map to pre-generated routing blocks. `code generate init` scaffolds a models file from your own omp instance (`omp models --json`, plus `omp usage --json` to spot the tier-scoped quota buckets that mark the spark and elite models) and `code generate` renders the catalog from it — see the -README quickstart. Two -honest limits: the tier assignments `init` derives (newest model per family, -then ranked by thinking ceiling, context and price) deserve a human look, and -the speed/ttft numbers it writes are placeholders unless you ask for -measured ones. `code generate init --bench` fills both from `omp bench --json` -— time to first token and output tok/s, one real API call per model, live -credentials required — and drops any model whose probe fails, which is the only -check that catches a model omp lists but your account cannot actually call. -Without it every model carries the identical placeholder pair, so the speed -meter those numbers drive is model-invariant: it moves with the thinking dial -and nothing else. +README quickstart. The tier assignments `init` derives (newest model per family, +then ranked by thinking ceiling, context and price) still deserve a human look. + +What no longer needs a caveat: `init` probes every candidate with `omp bench` +before it can become a rung, and that is mandatory rather than a flag. omp lists +models an account cannot actually call and nothing in the metadata says so — +`claude-mythos-5` reports `claude-fable-5`'s exact price, context window and +thinking range, and 404s here — so anything that does not come back with a clean +probe is dropped, as is anything missing from the report. The same pass supplies +the real speed/ttft, which used to be an identical placeholder pair on every +model, making the speed meter move with the thinking dial and nothing else. It +is one timed request per model, so `init` takes a minute and the figures are a +single sample rather than a steady benchmark. A verified file is marked +`probed: true`; `generate` refuses one that is not. ## Other honest caveats - oh-my-pi releases near-daily, and the `omp models --json` / `omp usage --json` - (and, with `--bench`, `omp bench --json`) schemas the generator reads carry no + / `omp bench --json` schemas the generator reads carry no stability guarantee — nor does the auth broker's snapshot/usage API the panel draws from. A scheduled compatibility check is planned: [#3](https://github.com/atyrode/code/issues/3). diff --git a/generate.go b/generate.go index 0f2f0dd..691fac1 100644 --- a/generate.go +++ b/generate.go @@ -90,16 +90,26 @@ func loadCatalogBytes(raw []byte, path string) (*catalog, error) { if len(doc.Content) == 0 { return nil, fmt.Errorf("%s: empty document", path) } - var modelsNode *yaml.Node + var modelsNode, probedNode *yaml.Node root := doc.Content[0] for i := 0; i+1 < len(root.Content); i += 2 { - if root.Content[i].Value == "models" { + switch root.Content[i].Value { + case "models": modelsNode = root.Content[i+1] + case "probed": + probedNode = root.Content[i+1] } } if modelsNode == nil { return nil, fmt.Errorf("%s: no `models:` mapping", path) } + // Reachability is a property of the file, not of the renderer. omp lists + // models an account cannot actually call and nothing in their metadata says + // so, and a rung that 404s breaks every profile it leads. `code generate + // init` sets this once it has probed; an offline scaffold leaves it false. + if probedNode == nil || probedNode.Value != "true" { + return nil, fmt.Errorf("%s: missing `probed: true` — these models were never verified as callable by your account. Re-run `code generate init --refresh`, which probes every model, or set `probed: true` yourself once you have confirmed each one", path) + } c := &catalog{models: map[string]catModel{}, levels: map[string][]int{}, ladder: map[string][5]string{"O": {}, "A": {}}} for i := 0; i+1 < len(modelsNode.Content); i += 2 { key := modelsNode.Content[i].Value @@ -752,9 +762,21 @@ func runGenerate(args []string) int { const generateHelp = `code generate — render the facet-grid catalog the TUI browses - code generate init [--from-json FILE] [--models-file OUT] - Scaffold a models file from your omp instance (runs 'omp models --json', - or reads FILE). Auto-guesses the pool/tier assignments — review them! + code generate init [--models-file OUT] [--refresh] [--from-json FILE] + Scaffold a models file from your omp instance. Reads 'omp models --json' + and 'omp usage --json', then probes every candidate with 'omp bench' to + confirm your account can actually call it — omp lists models it cannot, + and nothing in their metadata says so. Only verified models become rungs, + and the file is marked 'probed: true' so 'code generate' will accept it. + The probe makes one real request per model, so this takes a minute. + Derives the pool/tier assignments — review them! + + --refresh re-derive the tiers over an existing models file. Without it + init refuses to touch one, so a months-old scaffold keeps + naming models your providers have since retired. + --from-json read the model list from FILE instead of omp, and skip the + probe. Offline inspection only: the result is marked + 'probed: false' and 'code generate' will refuse to render it. code generate [--models-file FILE] [--out FILE|-] Render the catalog. Defaults: models file at diff --git a/generate_init.go b/generate_init.go index 7009826..ab4d414 100644 --- a/generate_init.go +++ b/generate_init.go @@ -362,25 +362,44 @@ func thinkingField(levels []string) string { return strings.Join(names, ",") } -// benchFact is one model's measured throughput, or a recorded probe failure. +// benchFact is one model's probe outcome. reachable means it answered; notFound +// means the provider says it does not exist for this account; anything else is +// unresolved — a transport, auth or rate failure that says nothing either way +// about entitlement, and so must not be guessed in either direction. type benchFact struct { speed, ttft float64 - ok bool + reachable bool + notFound bool + why string } -// ompBenchJSON measures models with omp's own benchmark; a var so tests can -// stub it. Three runs is enough to smooth a cold start without turning -// `--bench` into a coffee break. +// notFoundProbe matches the one failure that is genuinely disqualifying: the +// provider denying the model exists. claude-mythos-5 answers exactly this way +// while listing at claude-fable-5's price, context and thinking range. +var notFoundProbe = regexp.MustCompile(`(?i)not_found|model_not_found|no such model|unknown model|does not exist`) + +// ompBenchJSON probes models with omp's own benchmark; a var so tests can stub +// it. One short run per model: this is mandatory on every init and every +// first-run onboarding, and reachability is binary — extra runs or tokens would +// only steady a throughput figure at the cost of a longer wait. +// +// The prompt is overridden deliberately. omp's bundled bench prompt reads as +// cyber content to Anthropic's safety layer, and claude-fable-5 refuses it — +// which would have deleted a perfectly callable elite from the ladder. A probe +// that decides eligibility has to ask something nothing can object to. var ompBenchJSON = func(selectors []string) ([]byte, error) { args := append([]string{"bench"}, selectors...) - return exec.Command("omp", append(args, "--json", "--runs", "3")...).Output() + return exec.Command("omp", append(args, + "--json", "--runs", "1", "--max-tokens", "4", + "--prompt", "Reply with the single word: ok")...).Output() } -// runBench fills in speed and ttft, and doubles as a reachability probe. omp's -// catalog lists models the account cannot actually call — claude-mythos-5 is -// priced identically to claude-fable-5 and 404s here — and no metadata -// distinguishes them, so a model whose probe fails is recorded as unreachable -// and dropped from the ladder rather than merely left unmeasured. +// runBench sorts each model into the three outcomes benchFact describes and +// records throughput for the ones that answered. It deliberately does not +// collapse "refused" or "rate limited" into "unreachable": omp lists models an +// account cannot call (claude-mythos-5, at claude-fable-5's exact price), but a +// failed request is evidence of that only when the provider says the model does +// not exist. Everything else is unresolved, and the caller must not guess. func runBench(selectors []string) (map[string]benchFact, error) { raw, err := ompBenchJSON(selectors) if len(raw) == 0 { @@ -392,7 +411,12 @@ func runBench(selectors []string) (map[string]benchFact, error) { var parsed struct { Models []struct { Model string `json:"model"` - Average *struct { + Results []struct { + OK bool `json:"ok"` + Error string `json:"error"` + } `json:"results"` + Failures float64 `json:"failures"` + Average *struct { TTFTMs float64 `json:"ttftMs"` TokensPerSecond float64 `json:"tokensPerSecond"` } `json:"average"` @@ -410,27 +434,46 @@ func runBench(selectors []string) (map[string]benchFact, error) { // omp is not consistent about id casing across surfaces (usage scopes // report "GPT-5.3-Codex-Spark"), so key on a folded id at both ends. id = strings.ToLower(id) - if m.Average == nil { - out[id] = benchFact{} - continue + why := "" + for _, r := range m.Results { + if !r.OK { + if why = r.Error; why == "" { + why = "run failed without an error message" + } + break + } } - out[id] = benchFact{ - speed: math.Round(m.Average.TokensPerSecond*10) / 10, - ttft: math.Round(m.Average.TTFTMs/10) / 100, - ok: true, + switch { + case why == "" && m.Failures == 0 && len(m.Results) > 0 && m.Average != nil: + out[id] = benchFact{ + speed: math.Round(m.Average.TokensPerSecond*10) / 10, + ttft: math.Round(m.Average.TTFTMs/10) / 100, + reachable: true, + } + case notFoundProbe.MatchString(why): + out[id] = benchFact{notFound: true, why: why} + default: + if why == "" { + why = "incomplete probe report — no average or no runs recorded" + } + out[id] = benchFact{why: why} } } return out, nil } // scaffoldModels turns an `omp models --json` payload into models.yml content. -// Pure but for the optional quota probe, so the CLI and the first-run -// onboarding share it. bench, when non-nil, supplies measured speed/ttft. -func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { +// Pure but for the quota probe, so the CLI and the first-run onboarding share +// it. probe, when non-nil, is a reachability report: it supplies measured +// speed/ttft and decides which models may become rungs. nil means no probe ran +// (the --from-json path), and every listed model is taken on faith — output from +// that path is marked unprobed and `code generate` refuses it. +func scaffoldModels(raw []byte, probe map[string]benchFact) (string, error) { var parsed ompModels if err := json.Unmarshal(raw, &parsed); err != nil { return "", fmt.Errorf("parsing model list: %w", err) } + var unresolved []string byPool := map[string][]ompModel{} for _, m := range parsed.Models { pool := poolOf(m.Provider) @@ -438,11 +481,25 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { m.Cost.Input <= 0 || datedID.MatchString(m.ID) { continue } - // A model omp listed but could not actually call is worse than useless - // on the ladder: it looks top-spec and fails at launch. Only --bench - // knows, so only --bench can filter. - if f, probed := bench[strings.ToLower(m.ID)]; probed && !f.ok { - continue + // A model omp lists but the account cannot call is worse than useless on + // the ladder: it looks top-spec and 404s at launch, and no metadata + // distinguishes it (claude-mythos-5 lists at claude-fable-5's exact + // price, context and thinking range). Only a live probe knows, and only + // its verdict is actionable: a model the provider says does not exist is + // dropped, while a model that merely failed to answer is unresolved and + // must not be silently treated as either fine or absent. + if probe != nil { + f, seen := probe[strings.ToLower(m.ID)] + switch { + case f.notFound: + continue + case !seen: + unresolved = append(unresolved, m.ID+": missing from the probe report") + continue + case !f.reachable: + unresolved = append(unresolved, m.ID+": "+f.why) + continue + } } // Keep only thinking levels the generator's scale knows (omp can expose // provider-specific extras like "off"); without this the scaffold would @@ -460,6 +517,17 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { byPool[pool] = append(byPool[pool], m) } + // An unresolved probe is not a licence to proceed. Dropping these silently + // would write `probed: true` over a partial answer, and keeping them could + // crown a model that never replied — so refuse, name each one, and let the + // operator decide. Models the provider explicitly disowned are already gone + // above and deliberately absent from this list. + if len(unresolved) > 0 { + sort.Strings(unresolved) + return "", fmt.Errorf("the reachability probe came back inconclusive for %d model(s), so the ladder cannot be certified:\n %s\nre-run once the provider is answering, or pass --from-json to scaffold offline without a probe", + len(unresolved), strings.Join(unresolved, "\n ")) + } + specials := readSpecialTiers() type rung struct { m ompModel @@ -515,6 +583,42 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { specialTierNo = 0 } } + // Fallback when no bucket named an elite. The quota report is not always + // there to ask — it turns out to vary with the ambient auth environment, + // and the failure is silent and expensive: with the elite left on the + // ordinary ladder it becomes tier 3, so every routine "smart" request + // drains the scarce bucket that exists precisely to be spent on purpose. + // Price is legitimate evidence here. It says nothing about entitlement, + // which is why the reachability probe exists, but a model in its own + // price class is by definition not the everyday workhorse. + if specialTierNo < 0 && pool == "A" && len(cands) > 1 { + lead, next := cands[0], 0.0 + for _, m := range cands { + if m.Cost.Input > lead.Cost.Input { + lead = m + } + } + for _, m := range cands { + if m.ID != lead.ID && m.Cost.Input > next { + next = m.Cost.Input + } + } + if next > 0 && lead.Cost.Input >= 2*next { + specialTierNo, pickID = 4, lead.ID + } + } + // The bucket label comes from the quota scope when one named this lead. + // The price fallback has no scope to read, so it names the bucket after + // the tier it inferred — tier 4 is the elite window by construction. + specialBucket := "" + switch { + case pick != nil: + specialBucket = bucketName(pool, pick.tier) + case specialTierNo == 4: + specialBucket = bucketName(pool, "fable") + case specialTierNo == 0: + specialBucket = bucketName(pool, "spark") + } var elite ompModel var ladderCands []ompModel for _, m := range cands { @@ -525,7 +629,7 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { if specialTierNo == 4 { elite = m } - rungs[pool] = append(rungs[pool], rung{m, specialTierNo, bucketName(pool, pick.tier)}) + rungs[pool] = append(rungs[pool], rung{m, specialTierNo, specialBucket}) } // An elite defines the scarce, expensive class. Anything priced at or // above it is a sibling elite rather than a tier-3 workhorse, and must @@ -546,7 +650,11 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { if pool == "A" { name = "Anthropic" } - return "", fmt.Errorf("found %d usable %s model(s), need 3 (cheap/regular/smart) — code assumes both Anthropic and OpenAI are set up in omp", len(ladder), name) + hint := "code assumes both Anthropic and OpenAI are set up in omp" + if probe != nil { + hint += "; models the provider reported as non-existent were dropped by the probe" + } + return "", fmt.Errorf("found %d usable %s model(s), need 3 (cheap/regular/smart) — %s", len(ladder), name, hint) } for i, m := range ladder { rungs[pool] = append(rungs[pool], rung{m, i + 1, bucketName(pool, "")}) @@ -572,8 +680,23 @@ func scaffoldModels(raw []byte, bench map[string]benchFact) (string, error) { # # Re-render the catalog after any edit: code generate # Re-derive the tiers after a provider ships new models: code generate init --refresh -models: `) + // The marker is the whole point of probing: without it `code generate` + // refuses the file, so an offline scaffold can never quietly become live + // routing that names a model this account cannot call. + if probe != nil { + b.WriteString(`# Every model below answered a live probe, so 'code generate' will render it. +probed: true +`) + } else { + b.WriteString(`# NOT PROBED — scaffolded offline from --from-json, so no model here has been +# confirmed callable on your account and 'code generate' will refuse this file. +# Re-run 'code generate init --refresh' live, or flip this to true once you have +# checked each id yourself. +probed: false +`) + } + b.WriteString("models:\n") used := map[string]bool{} for _, pool := range []string{"O", "A"} { for _, r := range rungs[pool] { @@ -586,8 +709,8 @@ models: } used[key] = true speed, ttft := "50", "2.0" - note := " # placeholder — run `code generate init --refresh --bench` to measure" - if f, ok := bench[strings.ToLower(r.m.ID)]; ok { + note := " # placeholder — no probe ran (--from-json); re-run `code generate init` live to measure" + if f, ok := probe[strings.ToLower(r.m.ID)]; ok { speed, ttft, note = trimFloat(f.speed), trimFloat(f.ttft), "" } b.WriteString(fmt.Sprintf(` %s: @@ -628,7 +751,7 @@ func imageCapable(m ompModel) bool { func runGenerateInit(args []string) int { fromJSON, out := "", defaultModelsPath() - bench, refresh := false, false + refresh := false for i := 0; i < len(args); i++ { switch args[i] { case "--from-json": @@ -645,8 +768,6 @@ func runGenerateInit(args []string) int { return 2 } out = args[i] - case "--bench": - bench = true case "--refresh": refresh = true case "-h", "--help": @@ -674,17 +795,21 @@ func runGenerateInit(args []string) int { return 1 } + // Reachability is not optional: omp lists models an account cannot call and + // nothing in the metadata marks them, so a scaffold that skipped the probe + // can crown a model that 404s on every launch. The live path always probes; + // --from-json is an offline inspection path and labels its output as such. var facts map[string]benchFact - if bench { + if fromJSON == "" { sels, err := benchSelectors(raw) if err != nil { fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) return 1 } - fmt.Fprintf(os.Stderr, "benchmarking %d models — this makes real API calls and takes a while…\n", len(sels)) + fmt.Fprintf(os.Stderr, "probing %d models for reachability and speed — real API calls, takes a minute…\n", len(sels)) facts, err = runBench(sels) if err != nil { - fmt.Fprintf(os.Stderr, "code generate init: %v\n", err) + fmt.Fprintf(os.Stderr, "code generate init: the reachability probe failed (%v) — refusing to write a ladder that may name models your account cannot call; fix the provider credentials, or pass --from-json to scaffold offline\n", err) return 1 } } diff --git a/generate_test.go b/generate_test.go index 080f48b..82675f1 100644 --- a/generate_test.go +++ b/generate_test.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -12,7 +13,8 @@ import ( // optional spark/elite tiers present, an explicit quota bucket per model, and a // text-only model (the codex spark variants really are text-only) so the vision // role has something to route around. -const fixtureYML = `models: +const fixtureYML = `probed: true +models: luna: id: gpt-5.6-luna pool: O @@ -364,7 +366,7 @@ func TestCatalogWithoutOptionalTiers(t *testing.T) { func TestLoadCatalogValidation(t *testing.T) { cases := map[string]string{ - "missing ladder": "models:\n a:\n id: x\n pool: A\n tier: 1\n thinking: low→max\n", + "missing ladder": "probed: true\nmodels:\n a:\n id: x\n pool: A\n tier: 1\n thinking: low→max\n", "bad pool": strings.Replace(fixtureYML, "pool: O", "pool: X", 1), "dup tier": strings.Replace(fixtureYML, "tier: 2", "tier: 1", 1), "bad thinking": strings.Replace(fixtureYML, "low→max", "low-max", 1), @@ -569,15 +571,85 @@ func stubOmp(t *testing.T, usage string) { t.Cleanup(func() { ompUsageJSON = prev }) } -func scaffoldTo(t *testing.T, args ...string) (*catalog, string) { +// stubModels points the scaffolder's model-list probe at a fixed payload for +// the duration of a test, so the live init path never shells out to omp. +func stubModels(t *testing.T, models string) { + t.Helper() + prev := ompModelsJSON + ompModelsJSON = func() ([]byte, error) { return []byte(models), nil } + t.Cleanup(func() { ompModelsJSON = prev }) +} + +// benchProbe records what stubBench was asked to measure, so a test can prove +// the reachability probe actually fired rather than being skipped. +type benchProbe struct { + calls int + sels []string +} + +// stubBench points ompBenchJSON at a synthetic report that passes every +// selector it is handed, except those named in fail (matched on the whole +// selector or the bare id after the "provider/" prefix), which report the 404 +// not-found shape — the one failure runBench may read as unreachable and drop. +// It asserts benchSelectors' contract that every selector is provider-qualified +// (omp fuzzy-matches a bare id otherwise) and records the calls it served. +func stubBench(t *testing.T, fail ...string) *benchProbe { + t.Helper() + failSet := map[string]bool{} + for _, f := range fail { + failSet[f] = true + } + rec := &benchProbe{} + prev := ompBenchJSON + ompBenchJSON = func(sels []string) ([]byte, error) { + rec.calls++ + rec.sels = append(rec.sels, sels...) + rows := make([]string, 0, len(sels)) + for _, s := range sels { + if !strings.Contains(s, "/") { + t.Errorf("benchSelectors must hand omp provider-qualified selectors, got %q", s) + } + id := s[strings.LastIndexByte(s, '/')+1:] + if failSet[s] || failSet[id] { + // The 404 not-found shape: the provider disowns the model. That is + // the only failure runBench may read as unreachable, so these drop. + rows = append(rows, fmt.Sprintf(`{"model":%q,"results":[{"ok":false,"error":"404 {\"type\":\"error\",\"error\":{\"type\":\"not_found_error\",\"message\":\"model: %s\"}}"}],"failures":1,"average":null}`, s, id)) + continue + } + // A clean pass: every run ok, zero failures, a measured average. + rows = append(rows, fmt.Sprintf(`{"model":%q,"results":[{"ok":true}],"failures":0,"average":{"ttftMs":1404.2,"tokensPerSecond":48.94}}`, s)) + } + return []byte(`{"models":[` + strings.Join(rows, ",") + `]}`), nil + } + t.Cleanup(func() { ompBenchJSON = prev }) + return rec +} + +// passingProbe builds the reachability map a live bench would return when every +// listed model answers — the all-clear scaffoldModels gates rungs on. +func passingProbe(t *testing.T, modelsJSON string) map[string]benchFact { t.Helper() - dir := t.TempDir() - src := filepath.Join(dir, "omp.json") - out := filepath.Join(dir, "models.yml") - if err := os.WriteFile(src, []byte(initJSON), 0o644); err != nil { + var models ompModels + if err := json.Unmarshal([]byte(modelsJSON), &models); err != nil { t.Fatal(err) } - if code := runGenerateInit(append([]string{"--from-json", src, "--models-file", out}, args...)); code != 0 { + probe := map[string]benchFact{} + for _, m := range models.Models { + probe[strings.ToLower(m.ID)] = benchFact{speed: 48.9, ttft: 1.4, reachable: true} + } + return probe +} + +func scaffoldTo(t *testing.T, args ...string) (*catalog, string) { + t.Helper() + out := filepath.Join(t.TempDir(), "models.yml") + // The scaffolder's live path is mandatory-probe now. Stub the model list + // and a bench that passes every model, so init writes probed: true and the + // file loads back cleanly — all without a real API call. (--from-json is no + // longer usable here: it stamps probed: false, which loadCatalog rejects.) + stubModels(t, initJSON) + stubBench(t) + if code := runGenerateInit(append([]string{"--models-file", out}, args...)); code != 0 { t.Fatalf("runGenerateInit exit %d", code) } c, err := loadCatalog(out) @@ -654,35 +726,37 @@ func TestGenerateInitWithoutUsageProbe(t *testing.T) { } } -// --bench doubles as a reachability probe: omp lists claude-mythos-5 at -// claude-fable-5's exact price, but it 404s on accounts that do not have it, -// and no metadata distinguishes the two. +// The reachability probe is what stops omp's catalog quirks from becoming live +// routing: omp lists claude-mythos-5 at claude-fable-5's exact price, but mythos +// answers a 404 not-found on accounts that lack it and no metadata tells the two +// apart. A model the provider disowns must be dropped, and the scaffold that +// remains must still certify as probed. func TestGenerateInitBenchDropsUnreachableModels(t *testing.T) { stubOmp(t, "") - prev := ompBenchJSON - ompBenchJSON = func(sels []string) ([]byte, error) { - var rows []string - for _, s := range sels { - id := s[strings.LastIndexByte(s, '/')+1:] - if id == "claude-mythos-5" { - rows = append(rows, fmt.Sprintf(`{"model":%q,"average":null}`, s)) - continue - } - rows = append(rows, fmt.Sprintf(`{"model":%q,"average":{"ttftMs":1404.2,"tokensPerSecond":48.94}}`, s)) - } - return []byte(`{"models":[` + strings.Join(rows, ",") + `]}`), nil + stubModels(t, initJSON) + stubBench(t, "claude-mythos-5") + out := filepath.Join(t.TempDir(), "models.yml") + if code := runGenerateInit([]string{"--models-file", out}); code != 0 { + t.Fatalf("runGenerateInit exit %d", code) + } + c, err := loadCatalog(out) + if err != nil { + body, _ := os.ReadFile(out) + t.Fatalf("scaffold does not load back: %v\n%s", err, body) } - t.Cleanup(func() { ompBenchJSON = prev }) - - c, out := scaffoldTo(t, "--bench") for _, k := range c.keys { if c.models[k].ID == "claude-mythos-5" { - t.Error("a model whose bench probe failed must not reach the ladder") + t.Error("a model the provider 404s must not reach the ladder") } } body, _ := os.ReadFile(out) + // A not-found drop is not a partial probe: every survivor answered, so the + // file is certified rather than left unprobed. + if !strings.Contains(string(body), "probed: true") { + t.Errorf("dropping a 404 model must still yield a certified scaffold:\n%s", body) + } if strings.Contains(string(body), "placeholder") { - t.Error("--bench should replace the placeholder speed/ttft, not annotate them") + t.Error("a live probe should replace the placeholder speed/ttft, not annotate them") } if !strings.Contains(string(body), "speed: 48.9") || !strings.Contains(string(body), "ttft: 1.4") { t.Errorf("measured figures missing from scaffold:\n%s", body) @@ -691,13 +765,10 @@ func TestGenerateInitBenchDropsUnreachableModels(t *testing.T) { func TestGenerateInitRefresh(t *testing.T) { stubOmp(t, initUsage) - dir := t.TempDir() - src := filepath.Join(dir, "omp.json") - out := filepath.Join(dir, "models.yml") - if err := os.WriteFile(src, []byte(initJSON), 0o644); err != nil { - t.Fatal(err) - } - base := []string{"--from-json", src, "--models-file", out} + stubModels(t, initJSON) + stubBench(t) + out := filepath.Join(t.TempDir(), "models.yml") + base := []string{"--models-file", out} if code := runGenerateInit(base); code != 0 { t.Fatalf("first init exit %d", code) } @@ -720,3 +791,252 @@ func TestGenerateInitRefresh(t *testing.T) { t.Error("--refresh should re-derive the tiers from the current model list") } } + +// The probe marker is a hard gate: `code generate` must refuse a models file +// that was never verified as callable, and the rejection must name the +// remediation so the user is not left guessing. The remediation string is +// user-facing contract. +func TestLoadCatalogRequiresProbedMarker(t *testing.T) { + // fixtureYML is a healthy ladder carrying `probed: true`; strip or flip the + // marker to model the two unverified shapes a pre-gate scaffold produced. + body := strings.TrimPrefix(fixtureYML, "probed: true\n") + for _, tc := range []struct{ name, yml string }{ + {"no probed key", body}, + {"probed false", "probed: false\n" + body}, + } { + _, err := catalogFrom(t, tc.yml) + if err == nil { + t.Errorf("%s: an unverified models file must be rejected", tc.name) + continue + } + if !strings.Contains(err.Error(), "probed: true") { + t.Errorf("%s: rejection must name the `probed: true` remedy, got %v", tc.name, err) + } + } + // The marker present and true is the whole point: that file must load. + if _, err := catalogFrom(t, fixtureYML); err != nil { + t.Errorf("probed: true file rejected: %v", err) + } +} + +// A model the provider 404s is the one probe failure scaffoldModels may act on +// unilaterally: it drops the model and certifies the rest. claude-mythos-5 is +// the live trap — omp lists it at claude-fable-5's exact price, context and +// thinking, it 404s on accounts that lack it, and nothing in the metadata tells +// them apart, so only the probe can. +func TestScaffoldDropsNotFoundModels(t *testing.T) { + stubOmp(t, "") // scaffoldModels reads the usage report; keep it offline + probe := passingProbe(t, initJSON) + probe["claude-mythos-5"] = benchFact{notFound: true, why: "not_found_error"} + yml, err := scaffoldModels([]byte(initJSON), probe) + if err != nil { + t.Fatalf("a not-found model should drop cleanly, not error: %v", err) + } + if strings.Contains(yml, "claude-mythos-5") { + t.Errorf("a 404 model must be dropped from the scaffold:\n%s", yml) + } + if !strings.Contains(yml, "probed: true") { + t.Errorf("dropping a 404 model must still certify the scaffold:\n%s", yml) + } +} + +// The fable incident: omp's bundled bench prompt trips Anthropic's safety layer, +// so a perfectly callable claude-fable-5 comes back "Refusal (cyber)…". A +// refusal is evidence of nothing about entitlement, so it must NOT be collapsed +// into "unreachable" and silently delete the user's chosen elite. It has to +// surface as an unresolved error, while a genuine 404 (mythos) is still dropped +// — so the error names fable and never mentions mythos. +func TestScaffoldRefusalSurfacesNotDropped(t *testing.T) { + stubOmp(t, "") + probe := passingProbe(t, initJSON) + probe["claude-fable-5"] = benchFact{why: "Refusal (cyber): This request triggered restrictions"} + probe["claude-mythos-5"] = benchFact{notFound: true, why: "not_found_error"} + _, err := scaffoldModels([]byte(initJSON), probe) + if err == nil { + t.Fatal("a refused (unresolved) probe must not certify a ladder") + } + if !strings.Contains(err.Error(), "claude-fable-5") || !strings.Contains(err.Error(), "Refusal") { + t.Errorf("the refusal must surface, naming the model and reason: %v", err) + } + if strings.Contains(err.Error(), "claude-mythos-5") { + t.Errorf("a genuine 404 is dropped, not reported as unresolved: %v", err) + } +} + +// Anything the probe cannot resolve — a candidate missing from the report, or +// one that failed for a reason other than 404 — must refuse the whole scaffold +// rather than guess. Dropping silently would write probed: true over a partial +// answer; keeping would crown a model that never replied. +func TestScaffoldUnresolvedProbeRefuses(t *testing.T) { + stubOmp(t, "") + for _, tc := range []struct { + name string + mutate func(map[string]benchFact) + want string + }{ + {"missing from report", func(p map[string]benchFact) { delete(p, "claude-opus-5") }, "missing from the probe report"}, + {"failed but not 404", func(p map[string]benchFact) { p["claude-opus-5"] = benchFact{why: "rate_limited"} }, "rate_limited"}, + } { + probe := passingProbe(t, initJSON) + tc.mutate(probe) + _, err := scaffoldModels([]byte(initJSON), probe) + if err == nil { + t.Fatalf("%s: an unresolved probe must refuse the scaffold", tc.name) + } + if !strings.Contains(err.Error(), "claude-opus-5") || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: error must name the model and reason, got %v", tc.name, err) + } + } +} + +// The --from-json offline path (probe nil) is the one way to scaffold without a +// probe, so it must not become a bypass: it takes every listed model on faith +// yet stamps the file probed: false, and loadCatalog then refuses it. Both +// halves matter — retention keeps the path useful, the marker keeps it safe. +func TestScaffoldOfflineRetainsButMarksUnprobed(t *testing.T) { + stubOmp(t, initUsage) + yml, err := scaffoldModels([]byte(initJSON), nil) + if err != nil { + t.Fatalf("offline scaffold: %v", err) + } + // Retention: nothing is filtered on reachability, so the full ladder still + // scaffolds — the models a live probe would keep when everything passes. + for _, id := range []string{"claude-opus-5", "claude-fable-5", "gpt-5.6-sol"} { + if !strings.Contains(yml, id) { + t.Errorf("probe==nil must retain models; %s missing:\n%s", id, yml) + } + } + // Marking: yet the file is stamped unverified. + if !strings.Contains(yml, "probed: false") { + t.Errorf("offline scaffold must stamp probed: false:\n%s", yml) + } + // Bypass closure: an unverified file must not load. + p := filepath.Join(t.TempDir(), "models.yml") + if err := os.WriteFile(p, []byte(yml), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadCatalog(p); err == nil { + t.Error("loadCatalog must refuse a probed: false scaffold") + } else if !strings.Contains(err.Error(), "probed: true") { + t.Errorf("rejection should point at the remediation, got %v", err) + } +} + +// The live path must actually invoke the probe, not merely be capable of +// filtering. A unit test of scaffoldModels alone would miss an accidental nil +// facts at the call site, so assert through the stub that init handed omp +// provider-qualified selectors covering the models that can become rungs. +func TestGenerateInitLiveProbesModels(t *testing.T) { + stubOmp(t, initUsage) + stubModels(t, initJSON) + probe := stubBench(t) + out := filepath.Join(t.TempDir(), "models.yml") + if code := runGenerateInit([]string{"--models-file", out}); code != 0 { + t.Fatalf("runGenerateInit exit %d", code) + } + if probe.calls == 0 { + t.Fatal("live init must run the reachability probe") + } + // The probe has to reach the trap models, or an unreachable one slips + // through. Spot-check both pools, the elite, and the mythos look-alike. + for _, id := range []string{"claude-opus-5", "claude-fable-5", "claude-mythos-5", "gpt-5.6-sol"} { + found := false + for _, s := range probe.sels { + if strings.HasSuffix(s, "/"+id) { + found = true + break + } + } + if !found { + t.Errorf("probe selectors did not cover %s: %v", id, probe.sels) + } + } + // And the facts reached the file: a live scaffold is stamped probed: true. + body, _ := os.ReadFile(out) + if !strings.Contains(string(body), "probed: true") { + t.Errorf("live init must stamp probed: true:\n%s", body) + } +} + +// runBench sorts each probe row into exactly one of three outcomes, and the +// distinction is load-bearing: only the provider's 404 (notFound) may drop a +// model, a clean run makes it reachable, and everything else — refusals, rate +// limits, incomplete rows — is unresolved and must be neither dropped nor +// trusted. Collapsing a refusal into "unreachable" is what would have deleted +// claude-fable-5 from a real user's ladder. +func TestRunBenchClassifiesOutcomes(t *testing.T) { + prev := ompBenchJSON + t.Cleanup(func() { ompBenchJSON = prev }) + stub := func(row string) { + ompBenchJSON = func([]string) ([]byte, error) { return []byte(`{"models":[` + row + `]}`), nil } + } + + // Reachable: every run answered, so the measured figures come through. + stub(`{"model":"anthropic/x","results":[{"ok":true}],"failures":0,"average":{"ttftMs":1404.2,"tokensPerSecond":48.94}}`) + facts, err := runBench([]string{"anthropic/x"}) + if err != nil { + t.Fatalf("reachable: runBench: %v", err) + } + if f := facts["x"]; !f.reachable || f.notFound || f.speed != 48.9 || f.ttft != 1.4 { + t.Errorf("clean row should be reachable with measured figures, got %+v", f) + } + + // Not found: the provider disowns the model — the one droppable failure. + stub(`{"model":"anthropic/x","results":[{"ok":false,"error":"404 not_found_error: model does not exist"}],"failures":1,"average":null}`) + facts, err = runBench([]string{"anthropic/x"}) + if err != nil { + t.Fatalf("notFound: runBench: %v", err) + } + if f := facts["x"]; !f.notFound || f.reachable { + t.Errorf("a 404 row should be notFound, got %+v", f) + } + + // Unresolved: a refusal or an incomplete row says nothing about entitlement, + // so it is neither reachable nor notFound. + for _, tc := range []struct{ name, row string }{ + {"refusal", `{"model":"anthropic/x","results":[{"ok":false,"error":"Refusal (cyber): This request triggered restrictions"}],"failures":1,"average":null}`}, + {"failed run", `{"model":"anthropic/x","results":[{"ok":true},{"ok":false,"error":"stream closed"}],"failures":0,"average":{"ttftMs":1000,"tokensPerSecond":40}}`}, + {"nonzero failures", `{"model":"anthropic/x","results":[{"ok":true}],"failures":2,"average":{"ttftMs":1000,"tokensPerSecond":40}}`}, + {"no results", `{"model":"anthropic/x","results":[],"failures":0,"average":{"ttftMs":1000,"tokensPerSecond":40}}`}, + {"null average", `{"model":"anthropic/x","results":[{"ok":true}],"failures":0,"average":null}`}, + } { + stub(tc.row) + facts, err := runBench([]string{"anthropic/x"}) + if err != nil { + t.Fatalf("%s: runBench: %v", tc.name, err) + } + if f := facts["x"]; f.reachable || f.notFound { + t.Errorf("%s: must be unresolved (neither reachable nor notFound), got %+v", tc.name, f) + } + } +} + +// End to end, the live path must refuse rather than certify a partial probe: a +// refusal (the fable case) leaves no models file behind, so an unverified ladder +// can never reach `code generate`. +func TestGenerateInitRefusesUnresolvedProbe(t *testing.T) { + stubOmp(t, "") + stubModels(t, initJSON) + prev := ompBenchJSON + t.Cleanup(func() { ompBenchJSON = prev }) + ompBenchJSON = func(sels []string) ([]byte, error) { + rows := make([]string, 0, len(sels)) + for _, s := range sels { + // claude-fable-5 is callable, but omp's prompt trips a refusal; + // every other model answers cleanly. + if strings.HasSuffix(s, "/claude-fable-5") { + rows = append(rows, fmt.Sprintf(`{"model":%q,"results":[{"ok":false,"error":"Refusal (cyber): blocked under Anthropic's Usage Policy"}],"failures":1,"average":null}`, s)) + continue + } + rows = append(rows, fmt.Sprintf(`{"model":%q,"results":[{"ok":true}],"failures":0,"average":{"ttftMs":1404.2,"tokensPerSecond":48.94}}`, s)) + } + return []byte(`{"models":[` + strings.Join(rows, ",") + `]}`), nil + } + out := filepath.Join(t.TempDir(), "models.yml") + if code := runGenerateInit([]string{"--models-file", out}); code == 0 { + t.Error("init must fail when the probe is inconclusive") + } + if _, err := os.Stat(out); !os.IsNotExist(err) { + t.Errorf("no models file may be written from an unresolved probe (stat err: %v)", err) + } +} diff --git a/onboarding.go b/onboarding.go index ba01a97..f123e22 100644 --- a/onboarding.go +++ b/onboarding.go @@ -68,13 +68,26 @@ type obScanDoneMsg struct { type obGeneratedMsg struct{ err error } -// obScan reads the user's omp model list and scaffolds a models file from it. +// obScan reads the user's omp model list, verifies every candidate is actually +// callable on this account, and scaffolds a models file from the survivors. The +// probe is the slow part (a real request per model) but it is not optional: omp +// lists models an account cannot call, so skipping it can crown a rung that +// 404s on every launch. It runs here, inside the Tea command, so the spinner +// keeps painting. func obScan() tea.Msg { raw, err := ompModelsJSON() if err != nil { return obScanDoneMsg{err: fmt.Errorf("running `omp models --json`: %w", err)} } - yml, err := scaffoldModels(raw, nil) + sels, err := benchSelectors(raw) + if err != nil { + return obScanDoneMsg{err: err} + } + facts, err := runBench(sels) + if err != nil { + return obScanDoneMsg{err: fmt.Errorf("probing %d models for reachability: %w", len(sels), err)} + } + yml, err := scaffoldModels(raw, facts) return obScanDoneMsg{scaffold: yml, err: err} } @@ -235,7 +248,7 @@ func (o onboarding) View() string { clikit.StHead.Render("enter") + " continue · " + clikit.StHead.Render("q") + " quit", } case obScanning: - body = []string{o.spin.View() + " reading your omp model list…"} + body = []string{o.spin.View() + " reading your omp model list, then checking which models your account can actually call…"} case obReview: verb := "derived from your model list — sanity-check it" if o.existing { diff --git a/onboarding_test.go b/onboarding_test.go index 1d50ec3..892375e 100644 --- a/onboarding_test.go +++ b/onboarding_test.go @@ -29,6 +29,7 @@ func TestOnboardingScanFlow(t *testing.T) { t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "cfg")) t.Setenv("XDG_DATA_HOME", filepath.Join(t.TempDir(), "data")) stubOmp(t, initUsage) + stubBench(t) // obScan now probes every candidate; keep it offline and instant orig := ompModelsJSON ompModelsJSON = func() ([]byte, error) { return []byte(initJSON), nil } defer func() { ompModelsJSON = orig }() @@ -120,6 +121,8 @@ func TestOnboardingExistingModelsFile(t *testing.T) { func TestOnboardingErrorAndRetry(t *testing.T) { t.Setenv("XDG_CONFIG_HOME", filepath.Join(t.TempDir(), "cfg")) t.Setenv("XDG_DATA_HOME", filepath.Join(t.TempDir(), "data")) + stubOmp(t, initUsage) + stubBench(t) // the retry reaches obScan, which probes; keep it offline calls := 0 orig := ompModelsJSON ompModelsJSON = func() ([]byte, error) { From 60f27f7e77697fc477008db34fb5a7f1764ea803 Mon Sep 17 00:00:00 2001 From: Alex TYRODE Date: Sat, 25 Jul 2026 13:07:20 +0000 Subject: [PATCH 5/5] fix(generate): derive the agent legend instead of retyping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalog header names the roles the ● marker can appear on. It was a hand-typed list sitting beside genAgentRoles, so routing scout marked 414 rows the header denied existed. Derived from genAgentRoles in genRoleOrder, so the legend now reads in the same order as the rows beneath it and cannot drift again. A test asserts the legend and the marked rows name the same set - nothing covered that line before, which is why it went stale unnoticed. --- generate.go | 11 ++++++++++- generate_test.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/generate.go b/generate.go index 691fac1..d67d512 100644 --- a/generate.go +++ b/generate.go @@ -657,7 +657,16 @@ func (c *catalog) renderAdvisors() string { func (c *catalog) renderCatalog() string { var b strings.Builder b.WriteString("OMP generated routing — first-principles facet grid\n") - b.WriteString("bundled agents: designer librarian reviewer sonic task — ● marks an agent-backed role\n\n") + // Derived, not retyped: this legend named five agents while genAgentRoles + // carried six, so the grid marked a scout row the header denied existed. + var agents []string + for _, r := range genRoleOrder { + if genAgentRoles[r] { + agents = append(agents, r) + } + } + b.WriteString("bundled agents: " + strings.Join(agents, " ") + + " — ● marks an agent-backed role\n\n") b.WriteString(c.renderAdvisors() + "\n") b.WriteString(c.renderModelFacts() + "\n") hasSpark := c.ladder["O"][0] != "" diff --git a/generate_test.go b/generate_test.go index 82675f1..edb1acb 100644 --- a/generate_test.go +++ b/generate_test.go @@ -315,6 +315,51 @@ func TestScoutIsAgentBacked(t *testing.T) { } } +// The header legend tells the reader which roles the ● marker can appear on. +// It used to be a hand-typed list beside genAgentRoles, so adding scout marked +// a row the header denied existed. Derive-or-drift: the legend must name +// exactly the roles the grid actually marks. +func TestAgentLegendMatchesMarkedRoles(t *testing.T) { + c := fixtureCatalog(t) + out := c.renderCatalog() + var legend string + for _, l := range strings.Split(out, "\n") { + if strings.HasPrefix(l, "bundled agents: ") { + legend = strings.TrimPrefix(l, "bundled agents: ") + break + } + } + if legend == "" { + t.Fatal("catalog header must carry a bundled-agents legend") + } + named := map[string]bool{} + for _, f := range strings.Fields(legend) { + if f == "—" { + break + } + named[f] = true + } + marked := map[string]bool{} + for _, l := range strings.Split(out, "\n") { + if f := strings.Fields(l); len(f) > 1 && f[0] == "●" { + marked[f[1]] = true + } + } + if len(marked) == 0 { + t.Fatal("no ● rows rendered - the marker or the fixture regressed") + } + for r := range marked { + if !named[r] { + t.Errorf("role %q renders with ● but the legend omits it", r) + } + } + for r := range named { + if !marked[r] { + t.Errorf("legend names %q but no row carries the marker", r) + } + } +} + // The vision role feeds omp's image-describe fallback, so it must never lead on // a text-only model even when that model is the cheapest rung available. func TestVisionSkipsTextOnlyModels(t *testing.T) {