From 48f03fb0cf6e13224597a04ecad1993db9801ac2 Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Sun, 9 Aug 2026 12:14:00 +0200 Subject: [PATCH 1/4] Record each preset trial and service attempt in its own directory A trial was one appended line in a shared `trials.jsonl`, mirrored into the session directory by byte offset. An agent rewriting the file moved every offset, and the mirror then committed a torn fragment forever: one verified preset still carries an unparseable trial record. Service attempts had the same shape in `verifications.jsonl`. Every record is now written exactly once and reading is listing a directory: - `trials//` holds the trial's compiled `task.dstack.yml` and its `trial.json`; the presence of `trial.json` is what marks the trial completed, so in-flight and torn states are visible instead of corrupting. - `service//` holds each verification attempt's submitted YAML and a `verification.json` written when the attempt ends; an attempt directory without a result is one still in progress. - The byte-offset record mirrors are replaced by a stateless directory mirror that re-lists the source and copies changed files whole, scrubbed, and atomically. A torn read can never be committed; the next pass converges. Trial and attempt directories sort numerically, pinned past 9. The trial contract in the agent prompt is rewritten around the layout, with every per-trial file referenced by its full path and the write order stated where the files are defined. The listing shows the trial being worked on rather than the completed count while trialing, so `trialing (2/3)` cannot read as two finished; `verifying (3/3)` keeps the completed count. Validated end to end: a session on one RTX PRO 4500 produced three contract-exact trial records, a recorded verification attempt, and a saved preset that `dstack preset apply` deploys. Co-Authored-By: Claude Fable 5 --- .../_internal/cli/services/presets/agent.py | 17 +- .../_internal/cli/services/presets/output.py | 14 +- .../_internal/cli/services/presets/prompt.py | 2 +- .../presets/resources/system_prompt.md | 95 ++++--- .../_internal/cli/services/presets/session.py | 79 +++--- .../_internal/cli/services/presets/tail.py | 67 +++++ .../cli/services/presets/workspace.py | 21 +- .../cli/services/presets/test_agent.py | 250 +++++++++++++++--- .../cli/services/presets/test_output.py | 94 ++++--- 9 files changed, 465 insertions(+), 174 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/agent.py b/src/dstack/_internal/cli/services/presets/agent.py index 6dd1d8bbb..a1f87a734 100644 --- a/src/dstack/_internal/cli/services/presets/agent.py +++ b/src/dstack/_internal/cli/services/presets/agent.py @@ -23,6 +23,7 @@ print_preset_progress, ) from dstack._internal.cli.services.presets.tail import ( + _DirectoryMirror, _FileLineReader, _OffsetStore, _ProgressTailer, @@ -415,20 +416,16 @@ async def _session_tailers( offset_key="runs", echo=agent_session.echo, ), - _RecordMirror( - source=workspace.trials_path, - target=agent_session.trials_path, + _DirectoryMirror( + source=workspace.trials_dir, + target=agent_session.trials_dir, redacted_values=redacted_values, - offset_store=offset_store, - offset_key="trials", echo=agent_session.echo, ), - _RecordMirror( - source=workspace.verifications_path, - target=agent_session.verifications_path, + _DirectoryMirror( + source=workspace.service_dir, + target=agent_session.service_dir, redacted_values=redacted_values, - offset_store=offset_store, - offset_key="verifications", echo=agent_session.echo, ), ] diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index a15b63bdf..39a30c892 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -75,16 +75,20 @@ def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: return "".join(out) -def _format_trial_progress(session: Optional[dict[str, Any]]) -> str: +def _format_trial_progress(session: Optional[dict[str, Any]], *, in_flight: bool = False) -> str: """The ` (N/M)` suffix; stays outside the status markup to render in the - default color.""" + default color. While trialing, `N` is the trial being worked on rather than + the completed count, so `trialing (2/3)` cannot read as two finished.""" if not isinstance(session, dict): return "" trials = session.get("trials") trials_num = session.get("trials_num") if not isinstance(trials, dict) or not (trials.get("count") or isinstance(trials_num, int)): return "" - progress = str(trials.get("count") or 0) + count = trials.get("count") or 0 + if in_flight: + count = min(count + 1, trials_num) if isinstance(trials_num, int) else count + 1 + progress = str(count) if isinstance(trials_num, int): progress += f"/{trials_num}" return f" [secondary]({progress})[/]" @@ -180,7 +184,9 @@ def _add_session(table: Table, session: dict[str, Any], *, verbose: bool = False status_key = str(session.get("status", "")) if status_key == "running" and _verifying(session): status_key = "verifying" - status = _format_status(status_key) + _format_trial_progress(session) + status = _format_status(status_key) + _format_trial_progress( + session, in_flight=status_key == "running" + ) trials = session.get("trials") best = trials.get("best") if isinstance(trials, dict) else None # Nothing passed: fall back to the fastest attempt that did not. diff --git a/src/dstack/_internal/cli/services/presets/prompt.py b/src/dstack/_internal/cli/services/presets/prompt.py index e565fc2fd..7055c254f 100644 --- a/src/dstack/_internal/cli/services/presets/prompt.py +++ b/src/dstack/_internal/cli/services/presets/prompt.py @@ -18,7 +18,7 @@ # TODO: reintroduce a `# Resume` section in system_prompt.md once session resume -# (seeded from `runs.jsonl` and `trials.jsonl`) is designed. +# (seeded from `runs.jsonl` and the trial records) is designed. def get_preset_agent_system_prompt( user_prompt: Optional[str] = None, baseline: bool = False, diff --git a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md index d18187cb0..e5ad0bcb9 100644 --- a/src/dstack/_internal/cli/services/presets/resources/system_prompt.md +++ b/src/dstack/_internal/cli/services/presets/resources/system_prompt.md @@ -44,7 +44,7 @@ concurrencies instead of one.--> - `shared_prefix_tokens`: how many of `input_tokens` are identical in every request. `0` means every request is fully unique. - `baseline`: whether the first trial must be a baseline rather than an - optimization attempt; see `# Trials`. + optimization attempt; see `# Trials (Main Section)`. - `fleets`: use these existing `dstack` fleets only. Do not create, delete, apply, or edit fleets. - `env`: the environment variable names available to runs; the values are @@ -95,17 +95,13 @@ Files you are expected to maintain in the workspace root: `# Runs`. - `progress.jsonl`: progress messages, written through the `progress` helper; see `# Progress`. -- `trials.jsonl`: the append-only record of completed trials; see `# Trials`. -- `verifications.jsonl`: the append-only record of final service attempts; - see `# Final Service`. +- `trials/`: one directory per trial; see `# Trials (Main Section)`. +- `service/`: one directory per final service attempt; see `# Final Service`. - `final_report.json`: the final report; see `# Final Report`. -You may create any other working files (run YAML files, benchmark output, -notes) in the workspace, and only there: do not deliberately save files -elsewhere on this machine. Incidental writes made by the tools you run -(caches, temporary files, SSH configuration) are fine wherever those tools -keep them. Files inside running `dstack` tasks or services are not subject -to this rule. +On this machine, do not deliberately create, change, or delete files +outside the workspace. Inside running `dstack` tasks and services, write +whatever the work needs. # Runs @@ -165,7 +161,7 @@ how to get better performance than the previous trials. Sometimes it is worth continuing to improve a previous trial's idea, but when that risks settling into a local optimum, search for a substantially different approach rather than tweaking parameters further. - During the trials and experimentation aimed at the best performance, you may pick the hardware (the best available within the allowed `dstack` fleets), the model variant (only if `model` has `base`), the serving framework, the -Docker image and dependencies, the serving framework parameters, and -anything else within these constraints — except generating custom kernels, -patching drivers, patching serving framework source code, or P/D -disaggregation setups. - + + +Once `trials//task.dstack.yml` is written, write the corresponding +benchmark results (see `## Benchmark` for the structure) to +`trials//trial.json`: the presence of `trials//trial.json` is what +marks trial `` completed. The benchmark +may be skipped in one case only: you failed to make the configuration run +at all — a failed trial. `trials//task.dstack.yml` may be skipped in +one case only: you failed to get benchmark results at all. A trial is +also failed when its benchmark does not meet the constraints (see +`# Constraints`). When a trial that changed several things fails, be +mindful of which specific change was the root cause. `trials//trial.json` is one JSON object with these fields and no others: @@ -332,6 +337,40 @@ field of `trials//trial.json`; for the final benchmark, as enough. +## Patching Framework + +If the trial required patching the serving framework source code, +generating custom kernels, or patching drivers (see `# Constraints`), +when you save `trials//task.dstack.yml`, you must replicate these +exact patches. + +To do this, you must save the required patches in the +`trials//patches` directory (next to `trials//task.dstack.yml`), +and refer to them from `trials//task.dstack.yml` in the `files` +property. This will mount them inside the container. + +A patch is a unified diff against the file it changes. + +Example: + +```yaml +files: + - patches/vllm/model_executor/layers/fused_moe/fused_moe.py.patch:/patches/vllm/model_executor/layers/fused_moe/fused_moe.py.patch +``` + +Then, patches can be applied with `patch` (exactly when needed) from the +`commands`. + +Make sure to include only the patches that are required, and avoid +including unnecessary ones. + +This "patching framework" will allow you to replicate the fixes made +during the interactive SSH session via `trials//task.dstack.yml`. + +And, since writing `trials//task.dstack.yml` is done after the +interactive trial is completed, it's especially important to review that +patches are correct (and will exactly replicate the result). + # Task Usage Trials are done entirely using `dstack` tasks. For maximum efficiency, it is a @@ -375,7 +414,9 @@ SSH fleets can be treated as VM-based backends as they support both idle instanc # Final Service Once the trials are over, pick the best trial that has not been verified yet -and submit its configuration as a `dstack` service. Make it work with only +and submit its configuration as a `dstack` service. If there is no remaining +non-failed trial, pick the best failed trial that has a benchmark and has not +been verified yet. Make it work with only minor tweaks if needed; do not change the important decisions made during the trial. Set the service `model` name to the client-facing model name from `constraints.json` (see `# Constraints`). `model` is required: it also enables @@ -387,6 +428,10 @@ Record every attempt in its own directory `service//`, where `` is the attempt number: attempts are numbered from 1 in the order they are submitted. Immediately after submitting the service, create `service//` and write the submitted service YAML to `service//service.dstack.yml`. +If the service is based on a trial that required patches (see +`## Patching Framework`), save the required patches in +`service//patches` and refer to them from +`service//service.dstack.yml` in the `files` property. When the attempt ends, write `service//verification.json`, one JSON object with these fields and no others (values are illustrative): @@ -449,15 +494,17 @@ references in `final_report.json.service_yaml`; use environment variable names o # Final Report `final_report.json` may contain only `success`, `run_id`, `run_name`, -`service_yaml`, `base`, `model`, `context_length`, `benchmark`, and -`failure_summary`. +`service_yaml`, `trial`, `base`, `model`, `context_length`, `benchmark`, +and `failure_summary`. -On success, include exactly: +On success (even if you had to pick a failed trial because no non-failed +trial remained), include exactly: - `success`: `true` - `run_id`: the final verified service run ID - `run_name`: the final verified service run name - `service_yaml`: the full YAML of the verified final service +- `trial`: the `` of the verified trial - `base`: the base model repo, determined by the rules below - `model`: the exact repo/path loaded by the final service command - `context_length`: the largest context verified for the final service, as @@ -474,7 +521,7 @@ Set `final_report.json.base` as follows: to `model.repo`. - Do not infer `final_report.json.base` only from the repo name. -On failure, include exactly: +On failure (no trial verification was successful), include exactly: - `success`: `false` - `failure_summary`: the reason a preset could not be created and any change diff --git a/src/dstack/_internal/cli/services/presets/verify.py b/src/dstack/_internal/cli/services/presets/verify.py index 13a87990b..85b854323 100644 --- a/src/dstack/_internal/cli/services/presets/verify.py +++ b/src/dstack/_internal/cli/services/presets/verify.py @@ -65,11 +65,37 @@ def load_preset_agent_report( return report +def _rewrite_workspace_file_paths( + service: ServiceConfiguration, *, workspace_path: Path, session_path: Path +) -> None: + """Re-roots `files` local paths onto the session's mirrored record copies. + At submission they were resolved into the agent workspace, which is deleted + when the session ends; only `trials/` and `service/` are mirrored, so a + path outside them cannot outlive the workspace and fails the save.""" + workspace_root = workspace_path.resolve() + for mapping in service.files: + try: + relative = Path(mapping.local_path).resolve().relative_to(workspace_root) + except ValueError: + raise CLIError( + f"Claude final service file '{mapping.local_path}' is outside the agent workspace" + ) + target = session_path / relative + if relative.parts[:1] not in (("trials",), ("service",)) or not target.exists(): + raise CLIError( + f"Claude final service file '{mapping.local_path}' has no mirrored copy" + f" at '{target}'" + ) + mapping.local_path = str(target) + + def build_verified_preset( *, run: Run, preset_configuration: PresetConfiguration, report: AgentFinalReport, + workspace_path: Optional[Path] = None, + session_path: Optional[Path] = None, preset_id: Optional[str] = None, name: Optional[str] = None, ) -> Preset: @@ -91,11 +117,6 @@ def build_verified_preset( raise CLIError("Claude final report base does not match the requested model") elif report.model != preset_configuration.model.exact_repo: raise CLIError("Claude changed an exact model request") - if ( - preset_configuration.min_context_length is not None - and report.context_length < preset_configuration.min_context_length - ): - raise CLIError("Claude final service does not meet the requested context length") target_type = ( "gateway" if urlparse(run.service.url).scheme in {"http", "https"} else "server-proxy" @@ -111,6 +132,12 @@ def build_verified_preset( for key, value in preset_configuration.env.items(): if isinstance(value, EnvSentinel) and key in portable_service.env: portable_service.env[key] = value + if portable_service.files: + if workspace_path is None or session_path is None: + raise CLIError("Claude final service uses files but no workspace is attached") + _rewrite_workspace_file_paths( + portable_service, workspace_path=workspace_path, session_path=session_path + ) return build_preset( name=name, service=portable_service, @@ -119,6 +146,9 @@ def build_verified_preset( model=report.model, context_length=report.context_length, benchmark=benchmark, + trial=report.trial, + min_context_length=preset_configuration.min_context_length, + max_ttft=preset_configuration.max_ttft, preset_id=preset_id, ) diff --git a/src/tests/_internal/cli/preset_factories.py b/src/tests/_internal/cli/preset_factories.py index 6d5f6cad1..8a799bc8d 100644 --- a/src/tests/_internal/cli/preset_factories.py +++ b/src/tests/_internal/cli/preset_factories.py @@ -142,6 +142,7 @@ def get_successful_preset_report(run: Run) -> AgentFinalReport: run_id=run.id, run_name=run.run_spec.run_name, service_yaml="type: service", + trial=1, base="Qwen/Qwen3.5-27B", model="community/Qwen3.5-27B-GPTQ-Int4", context_length=32768, diff --git a/src/tests/_internal/cli/services/presets/test_apply.py b/src/tests/_internal/cli/services/presets/test_apply.py index 0597d3eab..d6c628042 100644 --- a/src/tests/_internal/cli/services/presets/test_apply.py +++ b/src/tests/_internal/cli/services/presets/test_apply.py @@ -27,7 +27,9 @@ def test_accepts_matching_base_model_and_context(self): _validate_preset_matches(preset, configuration=configuration) - def test_rejects_insufficient_context(self): + def test_warns_on_insufficient_context_instead_of_failing(self, capsys): + # The preset is chosen by ID and may be the best a session could verify; + # the shortfall is stated and the plan confirmation decides. preset = get_preset(preset_id="small", context_length=4096) configuration = PresetConfiguration( name="qwen", @@ -35,8 +37,11 @@ def test_rejects_insufficient_context(self): min_context_length=8192, ) - with pytest.raises(CLIError, match="context length"): - _validate_preset_matches(preset, configuration=configuration) + _validate_preset_matches(preset, configuration=configuration) + + output = capsys.readouterr().out + assert "verified for context length 4096" in output + assert "8192" in output def test_exact_request_matches_repo_and_client_facing_name(self): matching = get_preset(preset_id="matching") diff --git a/src/tests/_internal/cli/services/presets/test_verify.py b/src/tests/_internal/cli/services/presets/test_verify.py index b3b27e954..335394f9b 100644 --- a/src/tests/_internal/cli/services/presets/test_verify.py +++ b/src/tests/_internal/cli/services/presets/test_verify.py @@ -18,6 +18,7 @@ ) from dstack._internal.core.errors import CLIError from dstack._internal.core.models.envs import EnvSentinel +from dstack._internal.core.models.files import FilePathMapping from dstack._internal.core.models.profiles import ProfileParams from tests._internal.cli.preset_factories import ( get_running_service_run, @@ -71,6 +72,74 @@ def test_builds_portable_self_contained_preset(self): assert validation.benchmark.target.type == "server-proxy" assert validation.benchmark.client.type == "local" + def test_rewrites_file_paths_onto_the_mirrored_session_copies(self, tmp_path): + # `files` local paths resolve into the agent workspace at submission, and + # the workspace is deleted when the session ends; the preset must point at + # the session's mirrored copies or it cannot be applied later. + workspace = tmp_path / "session" / "workspace" / "w" + (workspace / "service" / "1" / "patches").mkdir(parents=True) + (workspace / "service" / "1" / "patches" / "moe.py.patch").write_text("--- a\n+++ b\n") + session = tmp_path / "session" + (session / "service" / "1" / "patches").mkdir(parents=True) + (session / "service" / "1" / "patches" / "moe.py.patch").write_text("--- a\n+++ b\n") + run = get_running_service_run() + run.run_spec.configuration.files = [ + FilePathMapping( + local_path=str(workspace / "service" / "1" / "patches"), path="/patches" + ) + ] + + preset = build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"} + ), + report=get_successful_preset_report(run), + workspace_path=workspace, + session_path=session, + ) + + assert preset.service.files[0].local_path == str(session / "service" / "1" / "patches") + # The run spec itself is untouched: only the preset copy is re-rooted. + assert run.run_spec.configuration.files[0].local_path == str( + workspace / "service" / "1" / "patches" + ) + + def test_rejects_a_file_without_a_mirrored_copy(self, tmp_path): + workspace = tmp_path / "session" / "workspace" / "w" + (workspace / "patches").mkdir(parents=True) # workspace root: not mirrored + session = tmp_path / "session" + run = get_running_service_run() + run.run_spec.configuration.files = [ + FilePathMapping(local_path=str(workspace / "patches"), path="/patches") + ] + + with pytest.raises(CLIError, match="no mirrored copy"): + build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"} + ), + report=get_successful_preset_report(run), + workspace_path=workspace, + session_path=session, + ) + + def test_rejects_files_when_no_workspace_is_attached(self, tmp_path): + run = get_running_service_run() + run.run_spec.configuration.files = [ + FilePathMapping(local_path=str(tmp_path / "patches"), path="/patches") + ] + + with pytest.raises(CLIError, match="no workspace is attached"): + build_verified_preset( + run=run, + preset_configuration=PresetConfiguration( + name="qwen-build", model={"base": "Qwen/Qwen3.5-27B"} + ), + report=get_successful_preset_report(run), + ) + def test_rejects_variant_for_exact_model_request(self): run = get_running_service_run() report = get_successful_preset_report(run).model_copy(update={"model": "other/model"}) From 4a4fa6567d4787315b7ea858ea47d0c0d3a26c0c Mon Sep 17 00:00:00 2001 From: Andrey Cheptsov Date: Mon, 10 Aug 2026 08:48:37 +0200 Subject: [PATCH 4/4] Mark constraint-breaching presets and scale the trial spark from zero A preset saved from a failed trial shows `*` next to its benchmark, the same mark a running session uses when it has only failed trials to show, computed from the requested constraints the preset now stores. In the trial spark, bars scale from zero so their heights compare as the numbers do, failed trials are red rather than gold, and gold marks the best result only while no trial meets the constraints. While trialing, the `(N/M)` progress counts the trial being worked on rather than the completed ones. Co-Authored-By: Claude Fable 5 --- .../_internal/cli/services/presets/output.py | 54 +++++++++++++------ .../cli/services/presets/test_output.py | 30 ++++++++--- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/src/dstack/_internal/cli/services/presets/output.py b/src/dstack/_internal/cli/services/presets/output.py index 39a30c892..21cc79f4f 100644 --- a/src/dstack/_internal/cli/services/presets/output.py +++ b/src/dstack/_internal/cli/services/presets/output.py @@ -36,9 +36,11 @@ def _verifying(session: dict[str, Any]) -> bool: def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: - """One glyph per trial, scaled within the run: the shape of the search. - A red `·` marks a trial that produced no benchmark at all; a yellow bar marks - one that measured but broke a constraint, since its number is real.""" + """One glyph per trial, scaled from zero: bar heights compare as the + numbers do, so the size of a gain is visible. A red `·` marks a trial + that produced no benchmark at all, and a red bar one that measured but + broke a constraint. Gold marks the best result while no trial meets the + constraints; green takes over once one does.""" if not isinstance(session, dict): return "" trials = session.get("trials") @@ -51,8 +53,12 @@ def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: values = [v for v in series if isinstance(v, (int, float))] if not values: return "·" * len(series) - low, high = min(values), max(values) - span = high - low + high = max(values) + passed = [v for v, f in zip(series, failed) if isinstance(v, (int, float)) and not f] + # The best trial is the answer the run found; everything else is context. Gold + # is that answer while none meets the constraints, so it gives way to green as + # soon as one does. + best = max(passed) if passed else max(values) out = [] for value, is_failed in zip(series, failed): if not isinstance(value, (int, float)): @@ -60,17 +66,13 @@ def _format_trial_spark(session: Optional[dict[str, Any]]) -> str: continue glyph = ( _SPARK_BLOCKS[-1] - if span <= 0 - else _SPARK_BLOCKS[round((value - low) / span * (len(_SPARK_BLOCKS) - 1))] + if high <= 0 + else _SPARK_BLOCKS[round(max(value, 0) / high * (len(_SPARK_BLOCKS) - 1))] ) - # The best trial is the answer the run found; everything else is context. - # Yellow, not red: the trial measured, its number is real, and only the - # constraint breach makes it unusable. Red is reserved for `·`, where - # nothing came back at all. if is_failed: - style = "gold1" + style = "gold1" if not passed and value >= best else "indian_red1" else: - style = "bold sea_green3" if value >= high else "secondary" + style = "bold sea_green3" if value >= best else "secondary" out.append(f"[{style}]{glyph}[/]") return "".join(out) @@ -285,8 +287,11 @@ def _add_preset( "": _format_trial_spark(creation), "CONSTRAINTS": format_preset_objective( preset, - min_context_length=(creation or {}).get("constraints", {}).get("min_context_length"), - max_ttft=(creation or {}).get("constraints", {}).get("max_ttft"), + # The preset carries what it was asked for; the creation record is the + # fallback for presets saved before it did. + min_context_length=preset.min_context_length + or (creation or {}).get("constraints", {}).get("min_context_length"), + max_ttft=preset.max_ttft or (creation or {}).get("constraints", {}).get("max_ttft"), verbose=verbose, ), "BENCHMARK": format_preset_benchmark(preset, verbose=verbose), @@ -336,6 +341,18 @@ def format_preset_objective( return f"[secondary]{' '.join(parts)}[/]" +def _breaches_constraints(preset: Preset) -> bool: + """Whether the verified benchmark misses what was asked for. A session that + found no compliant trial verifies its best failed one, so a preset can be + real, reproducible, and still fall short.""" + metrics = preset.validations[0].benchmark.metrics + if preset.max_ttft is not None and metrics.ttft_ms.p50 > preset.max_ttft: + return True + return preset.min_context_length is not None and ( + preset.context_length < preset.min_context_length + ) + + def format_preset_benchmark(preset: Preset, *, verbose: bool = False) -> str: benchmark = preset.validations[0].benchmark metrics = benchmark.metrics @@ -350,7 +367,12 @@ def format_preset_benchmark(preset: Preset, *, verbose: bool = False) -> str: f"ttft={_format_duration_ms(metrics.ttft_ms.p50)}", f"ctx={_format_token_count(preset.context_length)}", ] - return " ".join(parts) + text = " ".join(parts) + if _breaches_constraints(preset): + # Marked, not only dimmed: colour alone is not a signal. Same `*` a + # session row uses when it has nothing but failed trials to show. + return f"[secondary]*{text}[/]" + return text def _format_duration_ms(value: float) -> str: diff --git a/src/tests/_internal/cli/services/presets/test_output.py b/src/tests/_internal/cli/services/presets/test_output.py index 3dcfb3155..4f6d1c79f 100644 --- a/src/tests/_internal/cli/services/presets/test_output.py +++ b/src/tests/_internal/cli/services/presets/test_output.py @@ -393,9 +393,9 @@ def test_a_run_that_met_nothing_still_shows_what_it_measured(self): class TestFailedTrialSpark: - def test_a_trial_that_broke_a_constraint_is_yellow_but_still_charted(self): - # Its number is real, so it earns a bar; the breach makes it yellow, not - # red — red is reserved for a trial that produced nothing. + def test_a_trial_that_broke_a_constraint_is_charted_but_not_the_best(self): + # Its number is real, so it earns a bar rather than a `·`, and green goes + # to the best trial that meets the constraints even on a lower number. session = { "id": "ab12cd34", "status": "running", @@ -408,8 +408,26 @@ def test_a_trial_that_broke_a_constraint_is_yellow_but_still_charted(self): spark = output_module._format_trial_spark(session) - assert spark.count("gold1") == 1 - assert "indian_red1" not in spark + assert spark.count("indian_red1") == 1 + assert spark.count("sea_green3") == 1 + assert "gold1" not in spark assert "·" not in spark - # The failed trial is the highest number and must not be styled as best. + + def test_the_best_failed_trial_is_gold_while_none_passes(self): + # With nothing meeting the constraints, the best result so far is still + # what the run has to show; the rest are context. + session = { + "id": "ab12cd34", + "status": "running", + "trials": { + "count": 3, + "series": [100.0, 900.0, 300.0], + "failed": [True, True, True], + }, + } + + spark = output_module._format_trial_spark(session) + + assert spark.count("gold1") == 1 + assert spark.count("indian_red1") == 2 assert "sea_green3" not in spark