diff --git a/docs/batches.md b/docs/batches.md index 8ead13d0..5a8ad0bd 100644 --- a/docs/batches.md +++ b/docs/batches.md @@ -339,16 +339,18 @@ many; unmeasured assets remain eligible. A schema with no such class is refused **A class is left out of the prompt for either of two reasons, and both are published.** It does not admit `bbox`, so a detection has no shape to land as; or it declares a required attribute, -which a bare prediction carries no value for. Neither is visible in a run's outcome - a schema -whose `vehicle` requires a `color` completes a run, labels no vehicles, and says nothing about -why - so `GET /batches/{id}/pre-label` answers both halves before a run starts: `asked_classes` -is the prompt, and `excluded_classes` names the rest with every reason that holds against each. -Every class the pinned schema declares appears in exactly one of the two lists. It is derived -from the schema alone and needs no connection, so a dialog can name the classes before anybody -has chosen a model; a batch whose schema has no askable class at all is refused with the same -`SCHEMA_HAS_NO_DETECTABLE_CLASS` the launch answers, rather than reported as an empty prompt. At -a terminal `visionset batch pre-label` writes the same two lines to stderr before the first -forward pass. +which a bare prediction carries no value for. Neither is visible in the counters a run reports - +a schema whose `vehicle` requires a `color` completes a run, labels no vehicles, and the counts +say nothing about why - so `GET /batches/{id}/pre-label` answers both halves before a run starts: +`asked_classes` is the prompt, and `excluded_classes` names the rest with every reason that holds +against each. Every class the pinned schema declares appears in exactly one of the two lists. It +is derived from the schema alone and needs no connection, so a dialog can name the classes before +anybody has chosen a model; a batch whose schema has no askable class at all is refused with the +same `SCHEMA_HAS_NO_DETECTABLE_CLASS` the launch answers, rather than reported as an empty prompt. +At a terminal `visionset batch pre-label` writes the same two lines to stderr before the first +forward pass. The MCP tool `get_pre_label_plan` answers the same two halves, and there alone the +plan also travels *in* the outcome: `pre_label_batch` blocks until the run is done and returns it +under `plan`, so an agent that asked for nothing it expected never needs a second call. **What lands enters at `pre_labeled`, never `annotated`.** Nobody judged it, so it arrives in its own editable state rather than claiming to be somebody's work - see @@ -469,6 +471,8 @@ POST /batches/{id}/approve { "partition": … } → 200 BatchOut POST /batches/{id}/start → 200 BatchOut POST /batches/{id}/repin?allow_destructive= → 200 BatchOut POST /batches/{id}/complete → 200 BatchOut +GET /batches/{id}/pre-label → 200 PreLabelPlanOut, the prompt and + every class left out of it POST /batches/{id}/pre-label { "connection_id": …, "minimum_confidence": … } → 202 BackgroundJobOut POST /batches/{id}/promote → 200 AssetPage, the assets that entered GET /batches/{id}/jobs → 200 JobPage diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index c8a80957..54a44bbe 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -11,7 +11,7 @@ error envelope, and the three gate words. ## Always offered -49 tools, in the order an agent meets them: make a project, give it a schema, put images in it, work through them, promote, publish, export. +50 tools, in the order an agent meets them: make a project, give it a schema, put images in it, work through them, promote, publish, export. | Tool | Takes | What it does | | --- | --- | --- | @@ -33,6 +33,7 @@ error envelope, and the three gate words. | `get_batch` | `batch_id` | Read one batch: its state, its schema pin, its progress and its jobs. | | `approve_batch` | `batch_id`, `jobs_of`? | Freeze a batch, pin the project's active schema, and cut it into jobs. | | `start_batch` | `batch_id` | Open an approved batch for annotation. | +| `get_pre_label_plan` | `batch_id` | Which classes a pre-labeling run over this batch would ask a model about. | | `pre_label_batch` | `batch_id`, `connection`, `minimum_confidence`? | Ask a model to label every untouched asset in a batch. This blocks until it is done. | | `repin_batch` | `batch_id`, `allow_destructive`? | Move a batch's schema pin onto the project's *current* active version. | | `list_batch_assets` | `batch_id`, `limit`?, `offset`? | List a batch's assets, with the job each belongs to and its progress. | diff --git a/docs/mcp.md b/docs/mcp.md index b31be794..621c11fd 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -120,6 +120,7 @@ page groups them by what they are for. | `remove_batch_assets` | Take assets out of a draft. Deletes nothing. | | `approve_batch` | Freeze it, pin the schema, cut it into jobs. | | `start_batch` | Open it for annotation. | +| `get_pre_label_plan` | Which classes a run would ask about, and which it would leave out. | | `pre_label_batch` | Ask a model to label every untouched asset. Blocks until it is done. | | `repin_batch` | Move its schema pin onto the current active version. | | `list_batch_assets` | What is in it, paged, with each asset's job and progress. | @@ -307,7 +308,10 @@ the assets it fully entered, one commit per asset, so calling it again resumes w still untouched. `pre_label_batch` reports unmappable model labels as `regions_discarded` and mapped regions -without overlap with a measured asset as `regions_out_of_bounds`. +without overlap with a measured asset as `regions_out_of_bounds`, and the prompt it ran under +as `plan` — `asked_classes` beside `excluded_classes`, so a run that labeled nothing says +which classes it never asked about rather than leaving that to a second call. +`get_pre_label_plan` answers the same thing before the wait. There is therefore no ingest polling, and no `resume_ingest`. If a call is cut off part way, call `ingest` again - registration is idempotent on `(kind, path, extraction_fps)` and content diff --git a/src/visionset/inference/prelabel.py b/src/visionset/inference/prelabel.py index 575b8fc0..6fb302b4 100644 --- a/src/visionset/inference/prelabel.py +++ b/src/visionset/inference/prelabel.py @@ -165,9 +165,13 @@ class PreLabelPlan: """What a run over this schema would ask for, and what it would leave out. The two halves are derived together so they cannot disagree: every class the - schema declares appears in exactly one of them. + schema declares appears in exactly one of them, and the version they came + from travels with them so a surface reporting the plan need not resolve the + pin a second time. """ + #: The schema version both halves were derived from. A re-pin changes both. + schema_version: int #: The prompt, in the schema's own declaration order. asked: tuple[str, ...] #: The rest, each with why — empty when the whole schema is askable. @@ -201,7 +205,7 @@ def prompt_plan(schema: AnnotationSchema) -> PreLabelPlan: excluded.append(PreLabelExcludedClass(name=label_class.name, reasons=reasons)) else: asked.append(label_class.name) - return PreLabelPlan(asked=tuple(asked), excluded=tuple(excluded)) + return PreLabelPlan(schema_version=schema.version, asked=tuple(asked), excluded=tuple(excluded)) def detectable_classes(schema: AnnotationSchema) -> tuple[str, ...]: diff --git a/src/visionset/mcp/batches.py b/src/visionset/mcp/batches.py index e2b72420..9758c790 100644 --- a/src/visionset/mcp/batches.py +++ b/src/visionset/mcp/batches.py @@ -46,7 +46,13 @@ from pydantic import Field from visionset import wire -from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, pre_label +from visionset.inference import ( + DEFAULT_MINIMUM_CONFIDENCE, + PreLabelPlan, + pre_label, + prompt_plan, + require_detectable_schema, +) from visionset.kernel.domain import BySize, Partition from visionset.kernel.services import ( BatchService, @@ -267,6 +273,38 @@ def start_batch(batch_id: BatchRef) -> dict[str, Any]: return _batch_payload(workspace, started.id) +def get_pre_label_plan(batch_id: BatchRef) -> dict[str, Any]: + """Which classes a pre-labeling run over this batch would ask a model about. + + Call this before `pre_label_batch`. That call blocks for minutes and this one + is a single read, and what it answers decides whether the wait is worth it. + + **A run does not ask about every class the schema declares.** It asks about + the ones a bare box prediction can be written as, and `asked_classes` is that + list — it is the prompt itself, in the schema's own spelling. + + **`excluded_classes` names the rest, each with every reason it is left out.** + `no_bbox_geometry` means the class admits no box, so a detection has no shape + to land as. `required_attribute` means the class demands an attribute value, + and a model's answer carries none. Both can hold against one class, which is + why `reasons` is a list: a class told only that it admits no box, then given + one, would stay absent from the next run's prompt with nothing saying why. + + Every class the pinned schema declares appears in exactly one of the two + lists, and `schema_version` is the pin both were derived from — a re-pin + changes both. A schema with nothing askable at all is refused here rather + than answered with an empty prompt, exactly as `pre_label_batch` refuses + it — as is a batch that is not `in_annotation`. + + No connection is involved: the prompt is a property of the pinned schema + alone, so this answers the same lists whichever model is about to be asked. + """ + with opened_workspace() as workspace: + batch = BatchService(workspace).require_pre_labelable(identifier(batch_id, what="batch_id")) + schema = require_detectable_schema(workspace, batch) + return wire.pre_label_plan(prompt_plan(schema)) + + def pre_label_batch( batch_id: BatchRef, connection: ConnectionRef, @@ -328,10 +366,23 @@ class the schema declares that a box can be written as; an answer naming one or whose box classes each require an attribute a prediction cannot supply — has nowhere for a detection to land and is refused before anything runs. + `plan` in the result names both halves: `asked_classes` is what this run + actually asked about, and `excluded_classes` names every class of the pinned + schema it could not, each with every reason. `schema_version` is the pin + both were derived from. Read it whenever + `assets_labeled` is lower than expected — a run that asked about two of a + schema's five classes labels nothing under the other three, and the counters + alone cannot say so. `get_pre_label_plan` answers the same thing without + running anything. + Also refused before anything runs: a batch that is not `in_annotation`, a connection whose model answers places rather than words, and a deployment without the local runtime — with the install command in the message. """ + # Captured from the run rather than derived beside it: a plan read from the + # schema separately could differ from the one the run prompted with, and + # that it is the same list is the whole reason for reporting it. + seen: list[PreLabelPlan] = [] with opened_workspace() as workspace: resolved_connection = resolve_connection(workspace, connection) outcome = pre_label( @@ -339,6 +390,7 @@ class the schema declares that a box can be written as; an answer naming one batch_id=identifier(batch_id, what="batch_id"), connection_id=resolved_connection.id, minimum_confidence=minimum_confidence, + on_plan=seen.append, ) return { "assets_considered": outcome.assets_considered, @@ -348,6 +400,7 @@ class the schema declares that a box can be written as; an answer naming one "assets_skipped": outcome.assets_skipped, "regions_discarded": outcome.regions_discarded, "regions_out_of_bounds": outcome.regions_out_of_bounds, + "plan": wire.pre_label_plan(seen[0]), } diff --git a/src/visionset/mcp/main.py b/src/visionset/mcp/main.py index 3ea40613..1b790d03 100644 --- a/src/visionset/mcp/main.py +++ b/src/visionset/mcp/main.py @@ -98,6 +98,7 @@ (batches.get_batch, READS), (batches.approve_batch, WRITES), (batches.start_batch, WRITES), + (batches.get_pre_label_plan, READS), (batches.pre_label_batch, WRITES), (batches.repin_batch, WRITES), (batches.list_batch_assets, READS), diff --git a/src/visionset/server/models.py b/src/visionset/server/models.py index 138acd22..7c2491ce 100644 --- a/src/visionset/server/models.py +++ b/src/visionset/server/models.py @@ -1030,9 +1030,9 @@ class PreLabelPlanOut(BaseModel): excluded_classes: list[PreLabelExclusionOut] @classmethod - def of(cls, schema_version: int, plan: PreLabelPlan) -> Self: + def of(cls, plan: PreLabelPlan) -> Self: return cls( - schema_version=schema_version, + schema_version=plan.schema_version, asked_classes=list(plan.asked), excluded_classes=[ PreLabelExclusionOut(name=one.name, reasons=list(one.reasons)) diff --git a/src/visionset/server/routes/batches.py b/src/visionset/server/routes/batches.py index 6a3029af..752505bc 100644 --- a/src/visionset/server/routes/batches.py +++ b/src/visionset/server/routes/batches.py @@ -329,7 +329,7 @@ def pre_label_plan(workspace: WorkspaceDep, batch_id: UUID) -> PreLabelPlanOut: """ batch = BatchService(workspace).require_pre_labelable(batch_id) schema = require_detectable_schema(workspace, batch) - return PreLabelPlanOut.of(schema.version, prompt_plan(schema)) + return PreLabelPlanOut.of(prompt_plan(schema)) @router.post( diff --git a/src/visionset/wire/__init__.py b/src/visionset/wire/__init__.py index 1b6d9345..339d143b 100644 --- a/src/visionset/wire/__init__.py +++ b/src/visionset/wire/__init__.py @@ -57,8 +57,10 @@ # surfaces, it imports nothing from here, and what it owns is the fact this # module has no way to know — which model families this build can serve. The # alternative is spelling that mapping a second time, which is what every other -# rule in this file exists to prevent. -from visionset.inference import capabilities_of +# rule in this file exists to prevent. ``PreLabelPlan`` arrives the same way: +# the narrowing of a pinned schema to the classes a box can be written as is +# derived there, and every surface publishes it. +from visionset.inference import PreLabelPlan, capabilities_of from visionset.kernel.domain import ( Annotation, AnnotationJob, @@ -457,6 +459,25 @@ def pre_label_run(value: PreLabelRun) -> dict[str, Any]: } +def pre_label_plan(value: PreLabelPlan) -> dict[str, Any]: + """The prompt a pre-labeling run asks under, and every class left out of it. + + One spelling for the tool that answers the plan on its own and for the run + that reports the plan it ran under; two would be how an agent comes to see + ``excluded_classes`` under one and something else under the other. + ``schema_version`` is the pin both halves were derived from — a re-pin + changes both. + """ + return { + "schema_version": value.schema_version, + "asked_classes": list(value.asked), + "excluded_classes": [ + {"name": one.name, "reasons": [reason.value for reason in one.reasons]} + for one in value.excluded + ], + } + + def batch( value: Batch, counts: Mapping[AssetProgress, int], diff --git a/tests/cli/test_json_contract.py b/tests/cli/test_json_contract.py index 1569234f..87ef3cc7 100644 --- a/tests/cli/test_json_contract.py +++ b/tests/cli/test_json_contract.py @@ -60,9 +60,28 @@ from visionset import wire from visionset.formats._dummy import DummyExporter +from visionset.inference import PreLabelExcludedClass, PreLabelExclusionReason, PreLabelPlan from visionset.kernel.domain import AssetProgress, BackgroundJobState, PreLabelRun from visionset.server import models +#: A plan with both halves populated and one class carrying both reasons at +#: once, so neither list nor either reason goes unprojected. It is derived from +#: a schema rather than stored, which is why it is built here rather than in the +#: domain samples. +PRE_LABEL_PLAN = PreLabelPlan( + schema_version=SCHEMA_VERSION.version, + asked=("sign",), + excluded=( + PreLabelExcludedClass( + name="crossing", + reasons=( + PreLabelExclusionReason.NO_BBOX_GEOMETRY, + PreLabelExclusionReason.REQUIRED_ATTRIBUTE, + ), + ), + ), +) + # One row per pair: a label, the projected payload, and the wire model it must # agree with. Built eagerly — every projection runs at import, so a leaf that # does not encode fails collection rather than one parametrized case. @@ -166,6 +185,7 @@ wire.class_compatibility(EXPORT_COMPATIBILITY.classes[0]), models.ClassCompatibilityOut, ), + ("pre_label_plan", wire.pre_label_plan(PRE_LABEL_PLAN), models.PreLabelPlanOut), ] IDS = [label for label, _, _ in PAIRS] diff --git a/tests/mcp/test_batch_tools.py b/tests/mcp/test_batch_tools.py index a7e56806..79403f3f 100644 --- a/tests/mcp/test_batch_tools.py +++ b/tests/mcp/test_batch_tools.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any from uuid import uuid4 import pytest @@ -505,6 +506,20 @@ def _connection() -> str: return str(created["id"]) +#: A schema a run can only partly ask for: one class a box can be written as, +#: one that admits no box, and one failing both tests at once. The partial case +#: is the one the plan exists for — the total one is already refused. +MIXED_CLASSES: list[dict[str, Any]] = [ + {"name": "sign", "geometries": ["bbox"]}, + {"name": "centerline", "geometries": ["polyline"]}, + { + "name": "crossing", + "geometries": ["polygon"], + "attributes": [{"name": "painted", "kind": "boolean", "required": True}], + }, +] + + def test_pre_labeling_blocks_and_returns_what_it_wrote( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -522,10 +537,100 @@ def test_pre_labeling_blocks_and_returns_what_it_wrote( "assets_skipped": 0, "regions_discarded": 0, "regions_out_of_bounds": 0, + "plan": {"schema_version": 1, "asked_classes": ["sign"], "excluded_classes": []}, } assert payload(call("get_batch", batch_id=batch_id))["progress"]["pre_labeled"] == 2 +def test_a_run_that_labeled_nothing_says_what_it_asked_about( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The counters alone are the silence this key exists to end. + + The model answers `centerline`, which the prompt never asked for, so every + region is discarded and nothing is labeled. An agent reading only + `assets_labeled: 0` cannot tell that from a model that found nothing; the + prompt beside it is what makes the two distinguishable without a second call. + """ + _, batch_id, _job = open_batch(monkeypatch, tmp_path, count=2, classes=MIXED_CLASSES) + connection_id = _connection() + _predicting(monkeypatch, label="centerline") + + outcome = payload(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) + + assert outcome["assets_labeled"] == 0 + assert outcome["plan"]["asked_classes"] == ["sign"] + assert outcome["plan"]["excluded_classes"] == [ + {"name": "centerline", "reasons": ["no_bbox_geometry"]}, + {"name": "crossing", "reasons": ["no_bbox_geometry", "required_attribute"]}, + ] + + +def test_the_plan_names_the_prompt_and_every_class_left_out_of_it( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Both halves, in the schema's own order, and both reasons where both hold. + + `crossing` carries two: an agent told only that it admits no box would add + one and watch the class stay absent from the next run's prompt. + """ + _, batch_id, _job = open_batch(monkeypatch, tmp_path, count=2, classes=MIXED_CLASSES) + + plan = payload(call("get_pre_label_plan", batch_id=batch_id)) + + assert plan == { + "schema_version": 1, + "asked_classes": ["sign"], + "excluded_classes": [ + {"name": "centerline", "reasons": ["no_bbox_geometry"]}, + {"name": "crossing", "reasons": ["no_bbox_geometry", "required_attribute"]}, + ], + } + + +def test_the_plan_needs_no_connection_and_runs_no_model( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The prompt is a property of the pinned schema alone. + + Asked with no connection configured at all, which is what makes this the + call an agent can afford before committing minutes of inference. + """ + _, batch_id, _job = open_batch(monkeypatch, tmp_path, count=2) + + plan = payload(call("get_pre_label_plan", batch_id=batch_id)) + + assert plan["asked_classes"] == ["sign"] + assert plan["excluded_classes"] == [] + assert payload(call("get_batch", batch_id=batch_id))["progress"]["pre_labeled"] == 0 + + +def test_the_plan_refuses_a_schema_with_no_box_class( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """Refused rather than answered with an empty prompt. + + Pre-labeling this batch is impossible rather than merely unproductive, and + the run refuses with the same sentence — so an agent gets one answer from + both tools instead of an empty list from one and a refusal from the other. + """ + _, batch_id, _job = open_batch(monkeypatch, tmp_path, count=2, classes=[CENTERLINE]) + + refusal = error(call("get_pre_label_plan", batch_id=batch_id)) + + assert "no class that a box can be written as" in refusal["message"] + + +def test_the_plan_refuses_a_batch_that_is_not_being_annotated( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _, batch_id = ingested(monkeypatch, tmp_path, count=2) + + refusal = error(call("get_pre_label_plan", batch_id=batch_id)) + + assert refusal["message"] + + def test_a_batch_that_is_not_being_annotated_is_refused_and_writes_nothing( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: @@ -560,7 +665,9 @@ def test_a_schema_with_no_box_class_is_refused_and_writes_nothing( refusal = error(call("pre_label_batch", batch_id=batch_id, connection=connection_id)) - assert refusal["message"] + # The same sentence the plan tool refuses with, which is what makes one + # answer out of two tools a property a reader can check rather than a claim. + assert "no class that a box can be written as" in refusal["message"] assert payload(call("get_batch", batch_id=batch_id))["progress"]["review_pending"] == 0 diff --git a/tests/mcp/test_registration.py b/tests/mcp/test_registration.py index 3f24ad3c..67b22ee8 100644 --- a/tests/mcp/test_registration.py +++ b/tests/mcp/test_registration.py @@ -39,6 +39,7 @@ "get_batch", "approve_batch", "start_batch", + "get_pre_label_plan", "pre_label_batch", "repin_batch", "complete_batch",