diff --git a/docs/api.md b/docs/api.md index 68ffd2db..de95bd53 100644 --- a/docs/api.md +++ b/docs/api.md @@ -69,6 +69,7 @@ GET /batches/{batch_id} DELETE /batches/{batch_id} ?confirm=true POST /batches/{batch_id}/approve with a partition spec POST /batches/{batch_id}/start +GET /batches/{batch_id}/pre-label the classes a run would ask for POST /batches/{batch_id}/pre-label launch, poll the job POST /batches/{batch_id}/repin ?allow_destructive= POST /batches/{batch_id}/complete @@ -776,9 +777,10 @@ each is a decision somebody will otherwise try to "fix": - **Unknown keys pass.** `additionalProperties: false` constrains what the API *accepts*, not what it may one day *send*. A client that refused an added field would turn every backward-compatible release into a broken page. -- **An unknown member of an *open* vocabulary passes.** Six vocabularies carry - `x-visionset-open` in the spec — the four `allowed_actions` sets, `capabilities`, and - `SuggestionOut.parameters` — and the generated check for one accepts a member this client +- **An unknown member of an *open* vocabulary passes.** Seven vocabularies carry + `x-visionset-open` in the spec — the four `allowed_actions` sets, `capabilities`, + `SuggestionOut.parameters`, and the reasons a class is left out of a pre-label prompt — and + the generated check for one accepts a member this client never compiled against, exactly as it accepts an added field. Every other enum still refuses, and refuses the whole response with it: a value the client must *switch* on has no honest rendering to fall back to. The line between them is the field's shape. A vocabulary diff --git a/docs/batches.md b/docs/batches.md index 011604f4..8ead13d0 100644 --- a/docs/batches.md +++ b/docs/batches.md @@ -337,6 +337,19 @@ no overlap with a measured asset is discarded separately, and `regions_out_of_bo many; unmeasured assets remain eligible. A schema with no such class is refused up front; see [inference.md](inference.md#what-a-connection-can-be-asked-for). +**A class is left out of the prompt for either of two reasons, and both are published.** It does +not admit `bbox`, so a detection has no shape to land as; or it declares a required attribute, +which a bare prediction carries no value for. Neither is visible in a run's outcome - a schema +whose `vehicle` requires a `color` completes a run, labels no vehicles, and says nothing about +why - so `GET /batches/{id}/pre-label` answers both halves before a run starts: `asked_classes` +is the prompt, and `excluded_classes` names the rest with every reason that holds against each. +Every class the pinned schema declares appears in exactly one of the two lists. It is derived +from the schema alone and needs no connection, so a dialog can name the classes before anybody +has chosen a model; a batch whose schema has no askable class at all is refused with the same +`SCHEMA_HAS_NO_DETECTABLE_CLASS` the launch answers, rather than reported as an empty prompt. At +a terminal `visionset batch pre-label` writes the same two lines to stderr before the first +forward pass. + **What lands enters at `pre_labeled`, never `annotated`.** Nobody judged it, so it arrives in its own editable state rather than claiming to be somebody's work - see [annotations.md](annotations.md#provenance-is-the-models-own-rule-not-the-services). It is not diff --git a/docs/cli.md b/docs/cli.md index 230051a0..566769f8 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -358,8 +358,11 @@ interrupted run. The batch id goes to stdout. `DatasetService.promote` - it takes a *batch* id and derives the dataset, which is why it lives here. `pre-label BATCH_ID CONNECTION [--minimum-confidence FLOAT]` blocks and calls -`visionset.inference.pre_label` inline because a terminal has no dispatcher. Progress and the -summary are written to stderr; normal stdout contains `annotations_written`. With `--json`, the +`visionset.inference.pre_label` inline because a terminal has no dispatcher. Before the first +forward pass it names the classes it is about to ask for, and every class of the pinned schema it +is leaving out with the reason - a class the prompt omits labels nothing, and afterwards there is +only the silence to explain. Progress and the summary are written to stderr; normal stdout +contains `annotations_written`. With `--json`, the command prints the complete outcome instead, including `regions_discarded` for unmappable model labels and `regions_out_of_bounds` for mapped regions without overlap with a measured asset. diff --git a/docs/ui.md b/docs/ui.md index 86ae7041..a77d6cf1 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -750,6 +750,17 @@ asset already pre-labeled, annotated, skipped, awaiting review or accepted is pa label that lands enters at `pre_labeled`, never `annotated`, so an annotator corrects a machine's guess rather than inheriting it silently as their own work. +**The prompt is named, and so is everything left out of it.** A count of assets says nothing about +which classes a run will look for, so a schema whose `vehicle` requires an attribute completes a +run, labels no vehicles, and reads exactly like a run that should have labeled something. The +dialog reads `GET /batches/{id}/pre-label` when it opens and shows both halves: the classes it +asks for, and beside them each class it does not, with the reason - no box, or an attribute a +prediction cannot supply. The lists come off the wire rather than being derived from the pinned +schema in the browser, because the same narrowing decides what the run really prompts with. They +are shown again under a settled run's summary, which is where a run that labeled nothing is +actually read. A schema with no askable class at all refuses this read, and the dialog renders +that refusal and leaves `Start` dead rather than waiting for the press to produce it. + The route answers `202` with a background job, on the export and weight-download routes' contract, and the dialog polls it exactly as `ExportDialog` polls an export: nothing here waits for the run to finish, but nothing closes over an outcome unseen either. Every refusal the route diff --git a/frontend/ui-core/src/generated/api.ts b/frontend/ui-core/src/generated/api.ts index 54e2cff5..3b3a73f9 100644 --- a/frontend/ui-core/src/generated/api.ts +++ b/frontend/ui-core/src/generated/api.ts @@ -385,7 +385,30 @@ export interface paths { path?: never; cookie?: never; }; - get?: never; + /** + * Pre Label Plan + * @description The classes a pre-labeling run over this batch would ask a model for. + * + * A run's prompt is the batch's pinned schema narrowed to the classes a bare + * box prediction can be written as, and that narrowing is invisible once the + * run has finished: a schema whose `vehicle` class requires an attribute + * yields no vehicles and says nothing about why. Read this before launching to + * say which classes are in the prompt and which are not, with the reason + * beside each one. + * + * Derived, never stored, and free of the connection the launch needs — the + * prompt is a property of the schema alone, so this answers the same lists + * whichever model is about to be asked. + * + * A batch that no run could touch is refused rather than answered with empty + * lists, on the same terms the launch itself uses: an unknown batch is 404 + * `BATCH_NOT_FOUND`, a batch that is not `in_annotation` is 409 + * `BATCH_NOT_IN_ANNOTATION`, and a pinned schema with no class a box can be + * written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`. A batch open for + * annotation but pinning no schema version is a broken invariant and answers + * 500 `WORKSPACE_CORRUPT`. + */ + get: operations["pre_label_plan"]; put?: never; /** * Pre Label Batch @@ -3936,6 +3959,58 @@ export interface components { */ type: "polyline"; }; + /** + * PreLabelExclusionOut + * @description A class in a batch's pinned schema that a pre-labeling run will not ask for. + * + * Both reasons are properties of the class as the schema declares it, so the + * remedy is a schema edit: give the class `bbox` among its geometries, or drop + * the `required` flag from the attribute a prediction cannot supply. + * + * `reasons` can carry both at once, and every reason that holds is listed — + * a class told only that it admits no box, then given one, would otherwise + * stay silently absent from the next run's prompt. + */ + PreLabelExclusionOut: { + /** Name */ + name: string; + /** Reasons */ + reasons: components["schemas"]["PreLabelExclusionReason"][]; + }; + /** + * PreLabelExclusionReason + * @description Why a schema's class is not among the words a run asks for. + * + * Open because it travels as a list a client renders member by member rather + * than switches on: a release that finds a third way a class cannot hold a + * detection must not cost an older client the whole plan, and the class it + * names is visibly left out whether or not that client can word the reason. + * @enum {string} + */ + PreLabelExclusionReason: "no_bbox_geometry" | "required_attribute" | (string & {}); + /** + * PreLabelPlanOut + * @description The words a pre-labeling run over this batch would ask a model for. + * + * A run's prompt is the batch's pinned schema, narrowed to the classes a bare + * box prediction can be written as. That narrowing is invisible in the run's + * result — a schema whose `vehicle` class requires a `color` attribute yields + * no vehicles and no explanation — so it is published here, before a run + * starts, with the left-out classes named beside the asked-for ones. + * + * Every class the pinned schema declares appears in exactly one of the two + * lists, both in the schema's own declaration order. A batch whose schema has + * no askable class at all is refused rather than answered with an empty + * `asked_classes`: pre-labeling it is impossible, not merely unproductive. + */ + PreLabelPlanOut: { + /** Asked Classes */ + asked_classes: string[]; + /** Excluded Classes */ + excluded_classes: components["schemas"]["PreLabelExclusionOut"][]; + /** Schema Version */ + schema_version: number; + }; /** * PreLabelRequest * @description Which model should pre-label this batch, and how sure it has to be. @@ -5673,6 +5748,82 @@ export interface operations { }; }; }; + pre_label_plan: { + parameters: { + query?: never; + header?: never; + path: { + batch_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreLabelPlanOut"]; + }; + }; + /** @description Missing or invalid bearer token */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description No such resource */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The resource's state refuses this request */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The request payload is not processable */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description Unhandled server error, with an incident id */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The workspace is busy; retry after the header says */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + }; + }; pre_label_batch: { parameters: { query?: never; @@ -10821,5 +10972,6 @@ export interface KnownMembers { ConnectionAction: "download_weights" | "check_integrity" | "update" | "delete"; JobAction: "start" | "complete"; ModelCapability: "point_suggest" | "text_detect"; + PreLabelExclusionReason: "no_bbox_geometry" | "required_attribute"; SuggestParameter: "detail"; } diff --git a/frontend/ui-core/src/generated/checks.ts b/frontend/ui-core/src/generated/checks.ts index 8b29bb88..79f3c0b1 100644 --- a/frontend/ui-core/src/generated/checks.ts +++ b/frontend/ui-core/src/generated/checks.ts @@ -231,6 +231,15 @@ export const checkJobOut: Check = export const checkJobPage: Check = /*#__PURE__*/ object({ "items": [true, arrayOf(checkJobOut)], "total": [true, isInteger] } as const); +export const checkPreLabelExclusionReason: Check = + /*#__PURE__*/ openOneOf(["no_bbox_geometry", "required_attribute"] as const); + +export const checkPreLabelExclusionOut: Check = + /*#__PURE__*/ object({ "name": [true, isString], "reasons": [true, arrayOf(checkPreLabelExclusionReason)] } as const); + +export const checkPreLabelPlanOut: Check = + /*#__PURE__*/ object({ "asked_classes": [true, arrayOf(isString)], "excluded_classes": [true, arrayOf(checkPreLabelExclusionOut)], "schema_version": [true, isInteger] } as const); + export const checkProjectOut: Check = /*#__PURE__*/ object({ "description": [true, either([isString, isNull] as const)], "id": [true, isString], "name": [true, isString] } as const); @@ -412,6 +421,7 @@ export const checkListSchemaVersions = checkSchemaVersionPage; export const checkListSources = checkSourcePage; export const checkNextPendingAssets = checkAssetPage; export const checkPreLabelBatch = checkBackgroundJobOut; +export const checkPreLabelPlan = checkPreLabelPlanOut; export const checkPreviewSchemaChange = checkSchemaChangePreviewOut; export const checkPromoteBatch = checkAssetPage; export const checkPublishRelease = checkReleaseOut; diff --git a/frontend/ui-core/src/screens/PreLabelDialog.tsx b/frontend/ui-core/src/screens/PreLabelDialog.tsx index 7d3e0ba3..573fac92 100644 --- a/frontend/ui-core/src/screens/PreLabelDialog.tsx +++ b/frontend/ui-core/src/screens/PreLabelDialog.tsx @@ -23,6 +23,17 @@ * count shown here, is an upper bound on what a run will touch rather than an * exact one — which is why the string below says "up to". * + * ## The prompt is named, not described + * + * A run asks only for the classes a bare box prediction can be written as, and + * a count of assets says nothing about that — so a schema whose `vehicle` + * requires a `color` completes a run, labels no vehicles, and offers no reason. + * `usePreLabelPlan` fetches both halves and `PromptClasses` shows them, which is + * why a left-out class is visibly left out with its reason beside it. The lists + * are read off the wire rather than derived from the pinned schema here: the + * same narrowing decides what the run really prompts with, and a second copy of + * it in the browser is how a dialog comes to name a class no run asks about. + * * ## The job is a background one, and this dialog watches it — even one it did * not launch * @@ -80,12 +91,16 @@ import { SelectValue, } from "../primitives/Select"; import type { BadgeTone, Segment } from "./batchState"; +import type { KnownMembers } from "../generated/api"; import { batchKeys, useBackgroundJob, usePreLabelBatch, + usePreLabelPlan, type Batch, type BackgroundJob, + type PreLabelExclusion, + type PreLabelPlan, type PreLabelRun, } from "./queries"; @@ -270,6 +285,63 @@ function DoneSummary({ ); } +/** + * How each reason a class is left out reads. + * + * A `Record` over the vocabulary's *known* members, on `inferenceCatalog.ts`'s + * rule: a reason added to the kernel fails this build until its wording exists, + * rather than rendering an empty pair of parentheses. + */ +const EXCLUSION_PROSE: Record = { + no_bbox_geometry: "no box", + required_attribute: "requires an attribute a prediction cannot supply", +}; + +/** + * "vehicle (requires an attribute a prediction cannot supply)" — every reason + * this build can word. + * + * The vocabulary is open, so a newer server may name a reason this build has + * never compiled against. That one is dropped rather than printed raw, and a + * class whose every reason is unknown is still named: *which* class is missing + * from the prompt is the part that cannot be silently lost. + */ +function excludedProse(excluded: PreLabelExclusion): string { + const said = excluded.reasons + .map((one) => EXCLUSION_PROSE[one as KnownMembers["PreLabelExclusionReason"]]) + .filter((one) => one !== undefined); + return said.length === 0 ? excluded.name : `${excluded.name} (${said.join(", ")})`; +} + +/** + * The prompt, named — and beside it every class of this schema that is not in it. + * + * The count above says how many assets a run may touch and nothing about what it + * will look for, so a run that legitimately labels nothing reads exactly like a + * run that should have labeled something. Naming both halves answers that: a + * class missing from the prompt is visibly missing, and the reason sits next to + * it rather than in a schema the reader would have to go and reason about. + * + * Renders nothing at all while the read is in flight or has refused. A refusal + * here is `SCHEMA_HAS_NO_DETECTABLE_CLASS` — nothing in this schema is askable — + * and the dialog says that once, beside the dead press, rather than twice. + */ +function PromptClasses({ plan }: { readonly plan: PreLabelPlan | null }): JSX.Element | null { + if (plan === null) return null; + return ( +
+

+ Asks for {plan.asked_classes.join(", ")}. +

+ {plan.excluded_classes.length > 0 && ( +

+ Not asked for: {plan.excluded_classes.map(excludedProse).join("; ")}. +

+ )} +
+ ); +} + /** * `Start`, disabled, with `Edit these frames` beside it where there is * something to edit. The shared shape for every mode that has nothing left @@ -358,6 +430,9 @@ function PreLabelDialog({ () => remembered !== null && isSettled(remembered.state), ); const preLabel = usePreLabelBatch(batch?.id ?? ""); + // Read while the dialog is open and not before: the prompt is a property of + // the pinned schema, so a gallery that never opens this never asks for it. + const plan = usePreLabelPlan(batch?.id, batch?.schema_version, batch !== null); // The job this session launched if there is one, otherwise the batch's own // remembered run — watched by its id so a run still genuinely in flight, // started elsewhere, keeps polling here rather than sitting frozen. @@ -392,7 +467,11 @@ function PreLabelDialog({ // `Start`'s own twins — `Continue`, `Run again`, `Try again` — share this: a // launch with no untouched asset left is a guaranteed no-op, whichever mode // offers it. - const launchDisabled = running || active === undefined || !validConfidence || blocked; + // `plan.isError` is `SCHEMA_HAS_NO_DETECTABLE_CLASS` — the launch refuses on + // the same read, so pressing Start could only reproduce the refusal already + // on screen. + const launchDisabled = + running || active === undefined || !validConfidence || blocked || plan.isError; // The primitive the effect is actually a function of, not the object that // carries it — a `useBatch` refetch elsewhere on the page can mint a new // `Batch` with the same id, and that identity churn must not matter here. @@ -436,11 +515,9 @@ function PreLabelDialog({ Pre-label {batch?.name} - Asks the model for every class this batch’s schema admits as a box on its - own — never a polygon, polyline or tag, and never one that requires an attribute a - prediction cannot supply — over every asset nothing has touched yet. What it finds - lands pre-labeled and editable, never as somebody’s own - annotation. + Asks the model about every asset nothing has touched yet, under the classes named + below. What it finds lands pre-labeled and editable, never as + somebody’s own annotation.
@@ -495,6 +572,8 @@ function PreLabelDialog({
+ +

{untouchedSummary(untouched, total)}

@@ -526,7 +605,13 @@ function PreLabelDialog({ )} {mode === "done" && view !== null && ( - + <> + + {/* Again here, because this is where a run that labeled nothing is + read: the classes it never asked about are the answer, and the + dialog that named them before the run has long been closed. */} + + )} {mode === "stopped" && view !== null && ( @@ -555,6 +640,10 @@ function PreLabelDialog({ {blockedReason(view)} )} + {plan.isError && ( + {refusalProse(plan.error)} + )} + {preLabel.isError && ( {refusalProse(preLabel.error)} )} diff --git a/frontend/ui-core/src/screens/preLabel.test.tsx b/frontend/ui-core/src/screens/preLabel.test.tsx index 79c0e50b..ceef4d60 100644 --- a/frontend/ui-core/src/screens/preLabel.test.tsx +++ b/frontend/ui-core/src/screens/preLabel.test.tsx @@ -173,6 +173,26 @@ function preLabelRunOf(overrides: Record = {}): Record = {}): Record { + return { + schema_version: 3, + asked_classes: ["person", "car"], + excluded_classes: [ + { name: "vehicle", reasons: ["required_attribute"] }, + { name: "lane", reasons: ["no_bbox_geometry"] }, + { name: "crossing", reasons: ["no_bbox_geometry", "required_attribute"] }, + ], + ...overrides, + }; +} + /** The exact prose the kernel writes for each refusal `pre-label` can answer. */ const REFUSAL_MESSAGE: Record = { SCHEMA_HAS_NO_DETECTABLE_CLASS: @@ -193,6 +213,7 @@ interface Extra { readonly connections?: readonly Connection[]; readonly counts?: { readonly unannotated: number; readonly total: number }; readonly preLabel?: Answer; + readonly plan?: Answer; } function renderGallery(batchOverrides: Record = {}, extra: Extra = {}): void { @@ -213,6 +234,10 @@ function renderGallery(batchOverrides: Record = {}, extra: Extr body: { items: connections, total: connections.length }, }); on("POST", /\/pre-label$/, extra.preLabel ?? { status: 202, body: backgroundJobOf() }); + // Every test that opens this dialog reads the plan, so it is stubbed here + // rather than per test — and overridable, because the refusal is one of the + // things the dialog has to render. + on("GET", /\/pre-label$/, extra.plan ?? { status: 200, body: planOf() }); render(mount()); } @@ -256,6 +281,108 @@ it("says how many assets the run will touch", async () => { expect(await screen.findByText(/412 of 500/)).not.toBeNull(); }); +it("names the classes the run will ask for", async () => { + renderGallery({ allowed_actions: ["pre_label"] }); + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + + expect((await screen.findByTestId("prelabel-asked-classes")).textContent).toBe( + "Asks for person, car.", + ); +}); + +it("names every class it will not ask for, with the reason beside it", async () => { + renderGallery({ allowed_actions: ["pre_label"] }); + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + + // `crossing` carries both reasons: told only that it admits no box, somebody + // adds one and watches it stay absent from the next run's prompt. + expect((await screen.findByTestId("prelabel-excluded-classes")).textContent).toBe( + "Not asked for: vehicle (requires an attribute a prediction cannot supply); " + + "lane (no box); crossing (no box, requires an attribute a prediction cannot supply).", + ); +}); + +it("still names a class whose reason this build has never compiled against", async () => { + // The vocabulary is open, so a newer server may word a reason this build has + // no prose for. Which class is missing from the prompt must survive that — + // dropping the whole line would be the failure the plan exists to prevent. + renderGallery( + { allowed_actions: ["pre_label"] }, + { + plan: { + status: 200, + body: planOf({ + excluded_classes: [ + { name: "vehicle", reasons: ["something_this_build_never_saw"] }, + { name: "lane", reasons: ["no_bbox_geometry"] }, + ], + }), + }, + }, + ); + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + + expect((await screen.findByTestId("prelabel-excluded-classes")).textContent).toBe( + "Not asked for: vehicle; lane (no box).", + ); +}); + +it("says nothing about exclusions when the whole schema is askable", async () => { + renderGallery( + { allowed_actions: ["pre_label"] }, + { plan: { status: 200, body: planOf({ excluded_classes: [] }) } }, + ); + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + + await screen.findByTestId("prelabel-asked-classes"); + expect(screen.queryByTestId("prelabel-excluded-classes")).toBeNull(); +}); + +it("names the classes again beside a finished run's result", async () => { + on("GET", /\/background-jobs\//, { + status: 200, + body: backgroundJobOf({ + state: "succeeded", + processed: 48, + total: 48, + result: { + assets_labeled: 0, + annotations_written: 0, + regions_discarded: 0, + assets_skipped: 0, + stopped_early: false, + }, + }), + }); + renderGallery({ allowed_actions: ["pre_label"] }); + + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + await userEvent.click(await screen.findByRole("button", { name: /start/i })); + + // A run that labeled nothing is exactly where the left-out classes are the + // answer, and the dialog that named them before the run is long closed. + await screen.findByRole("button", { name: /edit these frames/i }); + expect((await screen.findByTestId("prelabel-excluded-classes")).textContent).toContain( + "vehicle (requires an attribute a prediction cannot supply)", + ); +}); + +it("shows the plan's own refusal and will not offer a run behind it", async () => { + renderGallery( + { allowed_actions: ["pre_label"] }, + { plan: refuses(409, "SCHEMA_HAS_NO_DETECTABLE_CLASS") }, + ); + await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); + + expect( + (await screen.findByTestId("prelabel-plan-error")).textContent, + ).toMatch(/no class that a box can be written as/i); + expect(screen.queryByTestId("prelabel-classes")).toBeNull(); + expect( + (screen.getByTestId("prelabel-submit") as HTMLButtonElement).disabled, + ).toBe(true); +}); + it("shows the refusal rather than swallowing it", async () => { renderGallery({ allowed_actions: ["pre_label"] }, { preLabel: refuses(409, "SCHEMA_HAS_NO_DETECTABLE_CLASS") }); await userEvent.click(await screen.findByRole("button", { name: /pre-label/i })); diff --git a/frontend/ui-core/src/screens/queries.ts b/frontend/ui-core/src/screens/queries.ts index 7da64a1c..1d3e00ce 100644 --- a/frontend/ui-core/src/screens/queries.ts +++ b/frontend/ui-core/src/screens/queries.ts @@ -70,6 +70,7 @@ import { checkListSources, checkCreateCorrectionBatch, checkPreLabelBatch, + checkPreLabelPlan, checkPreviewSchemaChange, checkPromoteBatch, checkPublishRelease, @@ -630,6 +631,8 @@ export type IngestFailure = components["schemas"]["IngestFailureOut"]; export type Batch = components["schemas"]["BatchOut"]; export type BatchPage = components["schemas"]["BatchPage"]; export type PreLabelRun = components["schemas"]["PreLabelRunOut"]; +export type PreLabelPlan = components["schemas"]["PreLabelPlanOut"]; +export type PreLabelExclusion = components["schemas"]["PreLabelExclusionOut"]; export const ingestKeys = { sources: (projectId: string) => ["projects", projectId, "sources"] as const, @@ -829,6 +832,11 @@ export const batchKeys = { batch: (batchId: string) => ["batches", batchId] as const, assets: (batchId: string) => ["batches", batchId, "assets"] as const, jobs: (batchId: string) => ["batches", batchId, "jobs"] as const, + // The pinned version is part of the key because the plan is a function of it + // and of nothing else: a re-pin must not leave a dialog naming the classes of + // a schema this batch no longer carries. + preLabelPlan: (batchId: string, schemaVersion: number | null) => + ["batches", batchId, "pre-label-plan", schemaVersion] as const, }; /** One request's worth. `docs/api.md`: paging bounds the response, not the read. */ @@ -1108,6 +1116,37 @@ export function usePreLabelBatch(batchId: string) { }); } +/** + * The classes a pre-labeling run would ask for, and the ones it would not. + * + * Served rather than derived here, though every input is on the wire: the same + * narrowing decides what a run actually prompts with, and a browser-side copy of + * it is how a dialog comes to name a class the run never asks about. `unwrap` + * surfaces `SCHEMA_HAS_NO_DETECTABLE_CLASS` as a refusal like any other — the + * dialog renders its prose and stops offering the launch. + * + * Read only while a dialog is open, and keyed by the pinned version so a re-pin + * asks again instead of answering from the previous schema's classes. + */ +export function usePreLabelPlan( + batchId: string | undefined, + schemaVersion: number | null | undefined, + enabled: boolean, +): UseQueryResult { + const client = useApiClient(); + return useQuery({ + queryKey: batchKeys.preLabelPlan(batchId ?? "none", schemaVersion ?? null), + enabled: enabled && batchId !== undefined, + queryFn: async () => + unwrap( + await client.GET("/batches/{batch_id}/pre-label", { + params: { path: { batch_id: batchId ?? "" } }, + }), + checkPreLabelPlan, + ), + }); +} + /** * One source, by id — the batch header's provenance line. * diff --git a/openapi.json b/openapi.json index 463415f1..e7ecd46b 100644 --- a/openapi.json +++ b/openapi.json @@ -3161,6 +3161,68 @@ "title": "PolylineGeometry", "type": "object" }, + "PreLabelExclusionOut": { + "description": "A class in a batch's pinned schema that a pre-labeling run will not ask for.\n\nBoth reasons are properties of the class as the schema declares it, so the\nremedy is a schema edit: give the class `bbox` among its geometries, or drop\nthe `required` flag from the attribute a prediction cannot supply.\n\n`reasons` can carry both at once, and every reason that holds is listed \u2014\na class told only that it admits no box, then given one, would otherwise\nstay silently absent from the next run's prompt.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "reasons": { + "items": { + "$ref": "#/components/schemas/PreLabelExclusionReason" + }, + "title": "Reasons", + "type": "array" + } + }, + "required": [ + "name", + "reasons" + ], + "title": "PreLabelExclusionOut", + "type": "object" + }, + "PreLabelExclusionReason": { + "description": "Why a schema's class is not among the words a run asks for.\n\nOpen because it travels as a list a client renders member by member rather\nthan switches on: a release that finds a third way a class cannot hold a\ndetection must not cost an older client the whole plan, and the class it\nnames is visibly left out whether or not that client can word the reason.", + "enum": [ + "no_bbox_geometry", + "required_attribute" + ], + "title": "PreLabelExclusionReason", + "type": "string", + "x-visionset-open": true + }, + "PreLabelPlanOut": { + "description": "The words a pre-labeling run over this batch would ask a model for.\n\nA run's prompt is the batch's pinned schema, narrowed to the classes a bare\nbox prediction can be written as. That narrowing is invisible in the run's\nresult \u2014 a schema whose `vehicle` class requires a `color` attribute yields\nno vehicles and no explanation \u2014 so it is published here, before a run\nstarts, with the left-out classes named beside the asked-for ones.\n\nEvery class the pinned schema declares appears in exactly one of the two\nlists, both in the schema's own declaration order. A batch whose schema has\nno askable class at all is refused rather than answered with an empty\n`asked_classes`: pre-labeling it is impossible, not merely unproductive.", + "properties": { + "asked_classes": { + "items": { + "type": "string" + }, + "title": "Asked Classes", + "type": "array" + }, + "excluded_classes": { + "items": { + "$ref": "#/components/schemas/PreLabelExclusionOut" + }, + "title": "Excluded Classes", + "type": "array" + }, + "schema_version": { + "title": "Schema Version", + "type": "integer" + } + }, + "required": [ + "schema_version", + "asked_classes", + "excluded_classes" + ], + "title": "PreLabelPlanOut", + "type": "object" + }, "PreLabelRequest": { "additionalProperties": false, "description": "Which model should pre-label this batch, and how sure it has to be.", @@ -5979,6 +6041,103 @@ } }, "/batches/{batch_id}/pre-label": { + "get": { + "description": "The classes a pre-labeling run over this batch would ask a model for.\n\nA run's prompt is the batch's pinned schema narrowed to the classes a bare\nbox prediction can be written as, and that narrowing is invisible once the\nrun has finished: a schema whose `vehicle` class requires an attribute\nyields no vehicles and says nothing about why. Read this before launching to\nsay which classes are in the prompt and which are not, with the reason\nbeside each one.\n\nDerived, never stored, and free of the connection the launch needs \u2014 the\nprompt is a property of the schema alone, so this answers the same lists\nwhichever model is about to be asked.\n\nA batch that no run could touch is refused rather than answered with empty\nlists, on the same terms the launch itself uses: an unknown batch is 404\n`BATCH_NOT_FOUND`, a batch that is not `in_annotation` is 409\n`BATCH_NOT_IN_ANNOTATION`, and a pinned schema with no class a box can be\nwritten as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`. A batch open for\nannotation but pinning no schema version is a broken invariant and answers\n500 `WORKSPACE_CORRUPT`.", + "operationId": "pre_label_plan", + "parameters": [ + { + "in": "path", + "name": "batch_id", + "required": true, + "schema": { + "format": "uuid", + "title": "Batch Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PreLabelPlanOut" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Missing or invalid bearer token" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "No such resource" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The resource's state refuses this request" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The request payload is not processable" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Unhandled server error, with an incident id" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The workspace is busy; retry after the header says" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Pre Label Plan", + "tags": [ + "batches" + ] + }, "post": { "description": "Ask a model to label every untouched asset in this batch, and answer at once.\n\nThe `pre_label` action. Labels land at `pre_labeled`, never at `annotated`:\nnobody judged them, so they arrive editable and correctable rather than\nclaiming to be somebody's work \u2014 and, being unjudged, they never reach the\nDataset until a person has taken them over.\n\n**Only assets nothing has touched \u2014 which is stronger than reading\n`unannotated`.** An asset already `pre_labeled`, annotated, skipped,\nawaiting review or accepted is passed over, and so is an `unannotated` one\nthat still carries annotations from an earlier round that was skipped and\nthen restored: that sequence deletes no labels, so progress alone does not\nprove an asset untouched. A run never writes over what a person did, and\nnever writes twice over what a model did \u2014 a second run extends an earlier\none onto whatever is still untouched rather than refreshing the labels it\nleft, so re-running with a lower confidence does not re-ask about a frame\nalready carrying a guess.\n\n**The batch's pinned schema is the prompt.** The model is asked for each class\nthe schema declares that a box can be written as; an answer naming one of\nthose classes, matched case-insensitively, is written under the schema's own\nspelling, and an answer naming none of them is discarded. A schema whose\nclasses are all polygons, polylines or tags \u2014 or whose box classes each\nrequire an attribute a prediction cannot supply \u2014 has nowhere for a\ndetection to land and is refused.\n\n**202, not 200.** A batch is hundreds of forward passes, so this follows the\nlaunch-and-poll contract the export and weight-download routes use: poll `GET\n/background-jobs/{id}` \u2014 the `Location` header names it \u2014 until `state` is\n`succeeded`, then re-read the batch's assets. Progress on the row is counted\nin assets.\n\n**Everything a caller can be told now is told now**, and no refusal creates a\njob \u2014 so a caller holding a job id holds one that will run. These refusals\nare about the request, and the caller can act on each. They are checked in\nthis order, and it is the order `pre_label` itself checks in, so a request\nwrong about the connection and the batch both always names the connection:\na connection whose weights have not arrived is 409\n`INFERENCE_CONNECTION_NOT_SET_UP`; a connection whose model answers places\nrather than words is 422 `UNSUPPORTED_PROMPT`; a batch that is not\n`in_annotation` is 409 `BATCH_NOT_IN_ANNOTATION`; a pinned schema with no\nclass a box can be written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`.\n\nTwo failures are about this installation rather than about the request, and\nanswer 500 carrying the message that says which: a machine without the\noptional local runtime is `LOCAL_INFERENCE_UNAVAILABLE` and carries the\nexact command that installs it, and a workspace whose records no longer\nhold together \u2014 a batch pinned to a schema version that is not stored \u2014 is\n`WORKSPACE_CORRUPT`. Neither is worth resending unchanged: there is no\nstate here a caller can change, so the remedy is the one the message names.\n\n**Asking twice joins the run already in flight rather than starting a second\none.** A request arriving while this batch has a pre-labeling run queued or\nrunning is answered with that run's id, so a double-click and a second tab\nwatch one run instead of paying for the same inference twice.", "operationId": "pre_label_batch", diff --git a/src/visionset/cli/batches.py b/src/visionset/cli/batches.py index 5a8ab3c8..5a75ac88 100644 --- a/src/visionset/cli/batches.py +++ b/src/visionset/cli/batches.py @@ -37,7 +37,12 @@ from visionset.cli._resolve import ProjectOption, resolve_project from visionset.cli._workspace import WorkspaceOption, opened_workspace from visionset.cli.inference import ConnectionArgument, _resolve -from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, pre_label +from visionset.inference import ( + DEFAULT_MINIMUM_CONFIDENCE, + PreLabelExclusionReason, + PreLabelPlan, + pre_label, +) from visionset.kernel.domain import AssetProgress, BySize, Partition from visionset.kernel.services import ( BatchService, @@ -188,6 +193,32 @@ def batch_start( _echo(started.id, started.state.value, json_out, wire.batch(started, counts, promoted=promoted)) +#: How each reason a class is left out of the prompt reads at a terminal. +#: Short, because they are joined inside a parenthesis beside the class name. +_EXCLUSION_PROSE: Final = { + PreLabelExclusionReason.NO_BBOX_GEOMETRY: "no box", + PreLabelExclusionReason.REQUIRED_ATTRIBUTE: "requires an attribute a prediction cannot supply", +} + + +def _announce(plan: PreLabelPlan) -> None: + """Say what the run is about to ask for, and what it is leaving out. + + Printed before the first forward pass because that is when it is still + actionable: a run that asks for two of a schema's five classes labels + nothing under the other three, and afterwards there is only the silence to + explain. + """ + note(f"Asking for {len(plan.asked)} class(es): {', '.join(plan.asked)}.") + if not plan.excluded: + return + left_out = "; ".join( + f"{one.name} ({', '.join(_EXCLUSION_PROSE[reason] for reason in one.reasons)})" + for one in plan.excluded + ) + note(f"Not asking for {len(plan.excluded)} class(es): {left_out}.") + + @batch_app.command("pre-label") def batch_pre_label( batch: BatchArgument, @@ -215,6 +246,7 @@ def batch_pre_label( batch_id=batch, connection_id=_resolve(connections, connection), minimum_confidence=minimum_confidence, + on_plan=_announce, on_progress=lambda done, total: note(f"Pre-labeling {done}/{total} asset(s)."), ) if json_out: diff --git a/src/visionset/inference/__init__.py b/src/visionset/inference/__init__.py index d0b55c32..67391d65 100644 --- a/src/visionset/inference/__init__.py +++ b/src/visionset/inference/__init__.py @@ -85,10 +85,14 @@ from visionset.inference.nms import DEFAULT_IOU_THRESHOLD, suppressed from visionset.inference.prelabel import ( DEFAULT_MINIMUM_CONFIDENCE, + PreLabelExcludedClass, + PreLabelExclusionReason, PreLabelOutcome, + PreLabelPlan, detectable_classes, no_detectable_class_message, pre_label, + prompt_plan, require_detectable_schema, unsupported_prompt_message, ) @@ -162,12 +166,16 @@ "DownloadSizes", "LocalSamProvider", "LocalTransformersProvider", + "PreLabelExcludedClass", + "PreLabelExclusionReason", "PreLabelOutcome", + "PreLabelPlan", "ProviderPool", "cache_root", "capabilities_of", "check_integrity", "detectable_classes", + "prompt_plan", "digest_of", "download", "download_size", diff --git a/src/visionset/inference/prelabel.py b/src/visionset/inference/prelabel.py index 26f83a25..575b8fc0 100644 --- a/src/visionset/inference/prelabel.py +++ b/src/visionset/inference/prelabel.py @@ -41,12 +41,14 @@ AssetProgress, Batch, GeometryType, + LabelClass, PredictionRequest, PredictionTarget, TextPrompt, media_type_of, ) from visionset.kernel.domain.geometry import geometry_intersects_asset +from visionset.kernel.domain.vocabulary import OpenVocabulary from visionset.kernel.errors import ( AssetNotWritable, SchemaHasNoDetectableClass, @@ -129,26 +131,89 @@ def no_detectable_class_message(schema_version: int) -> str: ) +class PreLabelExclusionReason(OpenVocabulary): + """Why a schema's class is not among the words a run asks for. + + Open because it travels as a list a client renders member by member rather + than switches on: a release that finds a third way a class cannot hold a + detection must not cost an older client the whole plan, and the class it + names is visibly left out whether or not that client can word the reason. + """ + + #: The class does not admit ``bbox``, so a detection has no shape to land as. + NO_BBOX_GEOMETRY = "no_bbox_geometry" + #: The class declares a required attribute. A model's answer carries no + #: attribute values, so a bare prediction has nothing to satisfy it with. + REQUIRED_ATTRIBUTE = "required_attribute" + + +@dataclass(frozen=True, slots=True) +class PreLabelExcludedClass: + """One class a run will not ask for, and every reason it will not. + + ``reasons`` is a sequence rather than a single value because both can hold + at once, and a caller told only the first would add ``bbox`` to a class and + watch it stay silently absent. + """ + + name: str + reasons: tuple[PreLabelExclusionReason, ...] + + +@dataclass(frozen=True, slots=True) +class PreLabelPlan: + """What a run over this schema would ask for, and what it would leave out. + + The two halves are derived together so they cannot disagree: every class the + schema declares appears in exactly one of them. + """ + + #: The prompt, in the schema's own declaration order. + asked: tuple[str, ...] + #: The rest, each with why — empty when the whole schema is askable. + excluded: tuple[PreLabelExcludedClass, ...] + + +def _exclusions(label_class: LabelClass) -> tuple[PreLabelExclusionReason, ...]: + """Every reason a bare prediction could not be written as this class.""" + reasons: list[PreLabelExclusionReason] = [] + if GeometryType.BBOX not in label_class.geometries: + reasons.append(PreLabelExclusionReason.NO_BBOX_GEOMETRY) + if any(attribute.required for attribute in label_class.attributes): + reasons.append(PreLabelExclusionReason.REQUIRED_ATTRIBUTE) + return tuple(reasons) + + +def prompt_plan(schema: AnnotationSchema) -> PreLabelPlan: + """Split a schema into the classes a run asks for and the ones it cannot. + + The prompt on its own is not enough to work from: both exclusions are + invisible in the result of a run, so somebody whose ``vehicle`` class + requires a ``color`` finds no vehicles labeled and nothing saying why. Naming + the left-out classes beside the asked-for ones puts the reason where the + absence is, and it is derived here so every surface says the same thing. + """ + asked: list[str] = [] + excluded: list[PreLabelExcludedClass] = [] + for label_class in schema.classes: + reasons = _exclusions(label_class) + if reasons: + excluded.append(PreLabelExcludedClass(name=label_class.name, reasons=reasons)) + else: + asked.append(label_class.name) + return PreLabelPlan(asked=tuple(asked), excluded=tuple(excluded)) + + def detectable_classes(schema: AnnotationSchema) -> tuple[str, ...]: """The class names a box can be written as, which are also the prompt. Public because the route asks it to refuse before enqueueing anything, and a second implementation of "which classes can hold a detection" is how a - request that was accepted comes to fail inside a worker. - - A class is excluded for either of two reasons: it does not admit ``bbox``, - or it declares a required attribute. A model's answer carries no attribute - values, so a class demanding one is not a class a bare prediction could ever - satisfy — excluding it here is what keeps that fact from surfacing as a - write that fails deep inside a run instead of as a batch this function never - offered. + request that was accepted comes to fail inside a worker. It reads + :func:`prompt_plan` for the same reason: the list a dialog shows and the + list a run prompts with are one derivation, never two. """ - return tuple( - label_class.name - for label_class in schema.classes - if GeometryType.BBOX in label_class.geometries - and not any(attribute.required for attribute in label_class.attributes) - ) + return prompt_plan(schema).asked def require_detectable_schema(workspace: WorkspaceService, batch: Batch) -> AnnotationSchema: @@ -184,6 +249,7 @@ def pre_label( batch_id: UUID, connection_id: UUID, minimum_confidence: float = DEFAULT_MINIMUM_CONFIDENCE, + on_plan: Callable[[PreLabelPlan], None] | None = None, on_progress: Callable[[int, int], None] | None = None, should_stop: Callable[[], bool] | None = None, pool: ProviderPool | None = None, @@ -210,6 +276,10 @@ def pre_label( passed over before the atomic write. ``PreLabelOutcome.regions_out_of_bounds`` says how many; unmeasured assets remain eligible. + ``on_plan`` is handed the prompt and the classes left out of it, once, after + every refusal has passed and before the first forward pass — what a surface + needs to say which classes a run will and will not ask for. + Raises: InferenceConnectionNotFound: no such connection. InferenceConnectionNotSetUp: a local connection whose weights are absent. @@ -233,7 +303,13 @@ def pre_label( batches = BatchService(workspace) batch = batches.require_pre_labelable(batch_id) schema = require_detectable_schema(workspace, batch) - phrases = detectable_classes(schema) + # Announced from inside the run rather than derived by the caller, so the + # plan a surface reports is the one this run is about to prompt with and the + # order the refusals above arrive in is left alone. + plan = prompt_plan(schema) + if on_plan is not None: + on_plan(plan) + phrases = plan.asked class_by_answer = _class_by_answer(phrases) jobs = batches.jobs(batch_id) diff --git a/src/visionset/server/models.py b/src/visionset/server/models.py index 40a77dae..138acd22 100644 --- a/src/visionset/server/models.py +++ b/src/visionset/server/models.py @@ -45,7 +45,12 @@ from fastapi import Query from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator -from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, capabilities_of +from visionset.inference import ( + DEFAULT_MINIMUM_CONFIDENCE, + PreLabelExclusionReason, + PreLabelPlan, + capabilities_of, +) from visionset.kernel.domain import ( DEFAULT_DETAIL, ActivityEntry, @@ -983,6 +988,59 @@ def of(cls, run: PreLabelRun) -> Self: ) +class PreLabelExclusionOut(BaseModel): + """A class in a batch's pinned schema that a pre-labeling run will not ask for. + + Both reasons are properties of the class as the schema declares it, so the + remedy is a schema edit: give the class `bbox` among its geometries, or drop + the `required` flag from the attribute a prediction cannot supply. + + `reasons` can carry both at once, and every reason that holds is listed — + a class told only that it admits no box, then given one, would otherwise + stay silently absent from the next run's prompt. + """ + + #: The class name, exactly as the pinned schema declares it. + name: str + #: Every reason this class is not askable, in a stable order: geometry + #: first, then attributes. + reasons: list[PreLabelExclusionReason] + + +class PreLabelPlanOut(BaseModel): + """The words a pre-labeling run over this batch would ask a model for. + + A run's prompt is the batch's pinned schema, narrowed to the classes a bare + box prediction can be written as. That narrowing is invisible in the run's + result — a schema whose `vehicle` class requires a `color` attribute yields + no vehicles and no explanation — so it is published here, before a run + starts, with the left-out classes named beside the asked-for ones. + + Every class the pinned schema declares appears in exactly one of the two + lists, both in the schema's own declaration order. A batch whose schema has + no askable class at all is refused rather than answered with an empty + `asked_classes`: pre-labeling it is impossible, not merely unproductive. + """ + + #: The pinned version these lists were derived from. A re-pin changes both. + schema_version: int + #: The prompt itself — the classes a run will ask the model about. + asked_classes: list[str] + #: The rest, each with why. Empty when the whole schema is askable. + excluded_classes: list[PreLabelExclusionOut] + + @classmethod + def of(cls, schema_version: int, plan: PreLabelPlan) -> Self: + return cls( + schema_version=schema_version, + asked_classes=list(plan.asked), + excluded_classes=[ + PreLabelExclusionOut(name=one.name, reasons=list(one.reasons)) + for one in plan.excluded + ], + ) + + # ``asset_ids`` is deliberately absent: membership is the paged listing's job, # and a batch of fifty thousand frames would otherwise ship its whole roll call # on every read of its name. ``schema_version`` is null exactly while the batch diff --git a/src/visionset/server/routes/batches.py b/src/visionset/server/routes/batches.py index 5a875c63..6a3029af 100644 --- a/src/visionset/server/routes/batches.py +++ b/src/visionset/server/routes/batches.py @@ -38,6 +38,7 @@ STUB_MODEL_ID, capabilities_of, not_set_up_message, + prompt_plan, require_detectable_schema, unsupported_prompt_message, ) @@ -80,6 +81,7 @@ JobPage, LimitQuery, OffsetQuery, + PreLabelPlanOut, PreLabelRequest, window, ) @@ -302,6 +304,34 @@ def create_correction_batch( ) +@router.get("/{batch_id}/pre-label", responses=documented(404, 409)) +def pre_label_plan(workspace: WorkspaceDep, batch_id: UUID) -> PreLabelPlanOut: + """The classes a pre-labeling run over this batch would ask a model for. + + A run's prompt is the batch's pinned schema narrowed to the classes a bare + box prediction can be written as, and that narrowing is invisible once the + run has finished: a schema whose `vehicle` class requires an attribute + yields no vehicles and says nothing about why. Read this before launching to + say which classes are in the prompt and which are not, with the reason + beside each one. + + Derived, never stored, and free of the connection the launch needs — the + prompt is a property of the schema alone, so this answers the same lists + whichever model is about to be asked. + + A batch that no run could touch is refused rather than answered with empty + lists, on the same terms the launch itself uses: an unknown batch is 404 + `BATCH_NOT_FOUND`, a batch that is not `in_annotation` is 409 + `BATCH_NOT_IN_ANNOTATION`, and a pinned schema with no class a box can be + written as is 409 `SCHEMA_HAS_NO_DETECTABLE_CLASS`. A batch open for + annotation but pinning no schema version is a broken invariant and answers + 500 `WORKSPACE_CORRUPT`. + """ + batch = BatchService(workspace).require_pre_labelable(batch_id) + schema = require_detectable_schema(workspace, batch) + return PreLabelPlanOut.of(schema.version, prompt_plan(schema)) + + @router.post( "/{batch_id}/pre-label", status_code=status.HTTP_202_ACCEPTED, diff --git a/tests/cli/test_batch_commands.py b/tests/cli/test_batch_commands.py index c0f3cfe3..7e5a2e4c 100644 --- a/tests/cli/test_batch_commands.py +++ b/tests/cli/test_batch_commands.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path from uuid import uuid4 @@ -12,14 +13,17 @@ jobs_of, ok, payload, + project, run, runner, started_batch, + stills, workspace, ) +from visionset.cli import batches as batch_commands from visionset.cli.main import app -from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE +from visionset.inference import DEFAULT_MINIMUM_CONFIDENCE, PreLabelExclusionReason from visionset.inference import prelabel as prelabel_module from visionset.kernel.domain import AssetPrediction, BboxGeometry, PredictedRegion from visionset.kernel.services import ( @@ -252,6 +256,79 @@ def test_pre_label_by_connection_id_writes_every_untouched_asset( assert payload(root, "batch", "list", "-p", name)["items"][0]["progress"]["pre_labeled"] == 6 +def _mixed_schema_batch(root: Path, tmp_path: Path) -> str: + """A started batch whose pinned schema a run can only partly ask for. + + Two classes a box can be written as, one that admits no box, and one failing + both tests at once — the ordinary partial case, which the default + ``SCHEMA_DOCUMENT`` (wholly askable) cannot show. + """ + document = { + "classes": [ + {"name": "sign", "geometries": ["bbox"]}, + {"name": "lane", "geometries": ["polygon"]}, + { + "name": "crossing", + "geometries": ["polygon"], + "attributes": [{"name": "painted", "kind": "boolean", "required": True}], + }, + {"name": "post", "geometries": ["bbox"]}, + ] + } + path = tmp_path / "mixed-schema.json" + path.write_text(json.dumps(document), encoding="utf-8") + project(root, "crossings") + ok(root, "schema", "apply", str(path), "--project", "crossings") + batch = ok( + root, "ingest", str(stills(tmp_path)), "--project", "crossings", "--batch-name", "stills" + ) + ok(root, "batch", "approve", batch) + ok(root, "batch", "start", batch) + return batch + + +def test_every_reason_a_class_is_left_out_has_a_sentence() -> None: + """A reason with no wording would print an empty pair of parentheses. + + The terminal has no build-time exhaustiveness check the way the browser's + `Record` does, so the vocabulary and its prose are compared here instead. + """ + assert set(batch_commands._EXCLUSION_PROSE) == set(PreLabelExclusionReason) + + +def test_pre_label_says_which_classes_it_asks_for( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + """The prompt, before the first forward pass rather than after the silence.""" + _, batch = started_batch(root, tmp_path) + + result = run(root, "batch", "pre-label", batch, _connection(root)) + + assert result.exit_code == 0, result.output + assert "Asking for 1 class(es): sign." in result.stderr + assert "Not asking for" not in result.stderr + + +def test_pre_label_names_every_class_it_leaves_out_and_why( + root: Path, tmp_path: Path, predicting: _FakePredictor +) -> None: + """A class absent from the prompt is visibly absent, with the reason beside it. + + `crossing` carries both reasons: told only that it admits no box, somebody + adds one and watches it stay absent from the next run. + """ + batch = _mixed_schema_batch(root, tmp_path) + + result = run(root, "batch", "pre-label", batch, _connection(root)) + + assert result.exit_code == 0, result.output + assert "Asking for 2 class(es): sign, post." in result.stderr + assert ( + "Not asking for 2 class(es): lane (no box); " + "crossing (no box, requires an attribute a prediction cannot supply)." in result.stderr + ) + + def test_pre_label_json_emits_the_complete_outcome( root: Path, tmp_path: Path, predicting: _FakePredictor ) -> None: diff --git a/tests/inference/test_prelabel.py b/tests/inference/test_prelabel.py index 19185a17..46b786c7 100644 --- a/tests/inference/test_prelabel.py +++ b/tests/inference/test_prelabel.py @@ -10,7 +10,15 @@ import pytest -from visionset.inference.prelabel import DEFAULT_MINIMUM_CONFIDENCE, detectable_classes, pre_label +from visionset.inference.prelabel import ( + DEFAULT_MINIMUM_CONFIDENCE, + PreLabelExcludedClass, + PreLabelExclusionReason, + PreLabelPlan, + detectable_classes, + pre_label, + prompt_plan, +) from visionset.kernel import BatchNotInAnnotation, SchemaHasNoDetectableClass, UnsupportedPrompt from visionset.kernel.domain import ( Annotation, @@ -539,6 +547,106 @@ def test_the_filter_excludes_a_class_a_prediction_cannot_satisfy() -> None: assert detectable_classes(schema) == ("post",) +def test_the_plan_names_every_left_out_class_with_its_reason() -> None: + """The two halves are one derivation, so a class lands in exactly one. + + ``SIGN`` is excluded for its required attribute alone — it admits bbox — + and ``LANE`` for its geometry alone, which is what makes the two reasons + distinguishable rather than a single "not askable" verdict. + """ + schema = AnnotationSchema(project_id=uuid4(), version=1, classes=(SIGN, POST, LANE)) + + plan = prompt_plan(schema) + + assert plan.asked == ("post",) + assert plan.excluded == ( + PreLabelExcludedClass(name="sign", reasons=(PreLabelExclusionReason.REQUIRED_ATTRIBUTE,)), + PreLabelExcludedClass(name="lane", reasons=(PreLabelExclusionReason.NO_BBOX_GEOMETRY,)), + ) + + +def test_a_class_failing_both_tests_reports_both_reasons() -> None: + """One reason would read as the whole answer. + + Told only that ``crossing`` admits no box, somebody adds bbox to it and + watches it stay silently absent — which is the exact shape of the confusion + the plan exists to end. + """ + crossing = LabelClass( + name="crossing", + geometries=(GeometryType.POLYGON,), + attributes=(Attribute(name="painted", kind="boolean", required=True),), + ) + schema = AnnotationSchema(project_id=uuid4(), version=1, classes=(POST, crossing)) + + plan = prompt_plan(schema) + + assert plan.asked == ("post",) + assert plan.excluded == ( + PreLabelExcludedClass( + name="crossing", + reasons=( + PreLabelExclusionReason.NO_BBOX_GEOMETRY, + PreLabelExclusionReason.REQUIRED_ATTRIBUTE, + ), + ), + ) + + +def test_the_prompt_is_the_plan_and_not_a_second_derivation() -> None: + """``detectable_classes`` reads the plan, so the two cannot disagree.""" + schema = AnnotationSchema( + project_id=uuid4(), version=1, classes=(SIGN, POST, LANE, *VEHICLE_CLASSES) + ) + + assert detectable_classes(schema) == prompt_plan(schema).asked + + +def test_a_run_announces_the_plan_it_is_about_to_prompt_with( + prelabel_fixture: Fixture, +) -> None: + """What a terminal prints before the first forward pass. + + Announced once, after the refusals and before any model runs, and carrying + the same phrases the provider is then asked for — a plan derived beside the + run rather than inside it could differ from what the run does. + """ + seen: list[object] = [] + + pre_label( + prelabel_fixture.workspace, + batch_id=prelabel_fixture.batch.id, + connection_id=prelabel_fixture.connection.id, + on_plan=seen.append, + pool=prelabel_fixture.pool, + ) + + assert len(seen) == 1 + plan = seen[0] + assert isinstance(plan, PreLabelPlan) + assert plan.asked == prelabel_fixture.pool.last_prompt.phrases + + +def test_a_refused_run_announces_no_plan(polygon_only_fixture: Fixture) -> None: + """The positive path above is what makes this absence mean anything. + + A schema with nothing askable never reaches the announcement, so a surface + cannot print a prompt for a run that will not happen. + """ + seen: list[object] = [] + + with pytest.raises(SchemaHasNoDetectableClass): + pre_label( + polygon_only_fixture.workspace, + batch_id=polygon_only_fixture.batch.id, + connection_id=polygon_only_fixture.connection.id, + on_plan=seen.append, + pool=polygon_only_fixture.pool, + ) + + assert seen == [] + + def test_a_bbox_class_with_a_required_attribute_is_refused_before_anything_loads( required_attribute_fixture: Fixture, ) -> None: diff --git a/tests/server/test_openapi_contract.py b/tests/server/test_openapi_contract.py index 60072ca6..ffd0cbf1 100644 --- a/tests/server/test_openapi_contract.py +++ b/tests/server/test_openapi_contract.py @@ -385,7 +385,7 @@ def test_a_vocabulary_is_open_exactly_when_its_shape_allows_it() -> None: assert wrong == [], "\n".join(wrong) -def test_the_open_set_is_the_six_the_client_was_generated_for() -> None: +def test_the_open_set_is_the_seven_the_client_was_generated_for() -> None: """The roster, so growing the set is a decision somebody makes on purpose. The gate above derives membership from shape and would stay green if the @@ -405,5 +405,6 @@ def test_the_open_set_is_the_six_the_client_was_generated_for() -> None: "ConnectionAction", "JobAction", "ModelCapability", + "PreLabelExclusionReason", "SuggestParameter", } diff --git a/tests/server/test_prelabel_route.py b/tests/server/test_prelabel_route.py index ec9d95c7..1bc466b2 100644 --- a/tests/server/test_prelabel_route.py +++ b/tests/server/test_prelabel_route.py @@ -37,6 +37,16 @@ #: Nothing here admits `bbox`, so a schema of only this has nowhere for a #: detection to land. POLYGON_ONLY = {"name": "lane", "geometries": ["polygon"]} +#: A box class failing both tests at once: no `bbox` among its geometries *and* +#: an attribute it demands. Neither reason alone is the whole answer for it. +DOUBLY_EXCLUDED = { + "name": "crossing", + "geometries": ["polygon"], + "attributes": [{"name": "painted", "kind": "boolean", "required": True}], +} +#: The ordinary schema — some classes a box can be written as, some not. The +#: partial case the plan exists for, since the total one is already refused. +MIXED = [DETECTABLE, POLYGON_ONLY, DOUBLY_EXCLUDED, {"name": "post", "geometries": ["bbox"]}] FETCHED_BYTES = 4_000_000_000 @@ -190,6 +200,14 @@ def attribute_gated_batch( return _open_batch(client, runner, tmp_path, monkeypatch, classes=[ATTRIBUTE_GATED]) +@pytest.fixture() +def mixed_batch( + client: TestClient, runner: InlineDispatcher, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> OpenBatch: + """A schema a run can partly ask for — two classes in, two left out.""" + return _open_batch(client, runner, tmp_path, monkeypatch, classes=MIXED) + + @pytest.fixture() def approved_batch( client: TestClient, runner: InlineDispatcher, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -350,6 +368,80 @@ def test_a_batch_that_is_not_being_annotated_is_refused_and_queues_nothing( assert _pre_label_job_count(client) == 0 +# --- the plan a launch would run ------------------------------------------------- + + +def test_the_plan_names_the_prompt_and_every_class_left_out_of_it( + client: TestClient, mixed_batch: OpenBatch +) -> None: + """Both lists, in the schema's own order, and both reasons where both hold. + + `crossing` carries two: told only that it admits no box, somebody adds one + and watches it stay absent. + """ + response = client.get(f"/batches/{mixed_batch.id}/pre-label") + + assert response.status_code == 200, response.text + body = response.json() + assert body["schema_version"] == 1 + assert body["asked_classes"] == ["sign", "post"] + assert body["excluded_classes"] == [ + {"name": "lane", "reasons": ["no_bbox_geometry"]}, + {"name": "crossing", "reasons": ["no_bbox_geometry", "required_attribute"]}, + ] + + +def test_the_plan_needs_no_connection_to_answer(client: TestClient, mixed_batch: OpenBatch) -> None: + """The prompt is a property of the schema alone. + + Read as a plain GET with nothing about a model in it — which is what lets a + dialog show the classes before anybody has chosen what to run. + """ + asked = client.get(f"/batches/{mixed_batch.id}/pre-label").json()["asked_classes"] + + assert asked == ["sign", "post"] + assert _pre_label_job_count(client) == 0 + + +def test_a_wholly_askable_schema_leaves_nothing_out( + client: TestClient, in_annotation_batch: OpenBatch +) -> None: + response = client.get(f"/batches/{in_annotation_batch.id}/pre-label") + + assert response.status_code == 200, response.text + assert response.json()["excluded_classes"] == [] + + +def test_the_plan_refuses_a_schema_with_no_box_class( + client: TestClient, polygon_only_batch: OpenBatch +) -> None: + """Refused rather than answered with an empty prompt. + + Pre-labeling this batch is impossible, not merely unproductive, and the + launch says so with this same code — so the dialog can stop before the press + rather than after it. + """ + response = client.get(f"/batches/{polygon_only_batch.id}/pre-label") + + assert response.status_code == 409 + body = response.json() + assert body["code"] == "SCHEMA_HAS_NO_DETECTABLE_CLASS" + assert body["message"] + + +def test_the_plan_refuses_a_batch_that_is_not_being_annotated( + client: TestClient, draft_batch: OpenBatch +) -> None: + response = client.get(f"/batches/{draft_batch.id}/pre-label") + + assert response.status_code == 409 + assert response.json()["code"] == "BATCH_NOT_IN_ANNOTATION" + + +def test_the_plan_of_an_unknown_batch_is_not_found(client: TestClient) -> None: + assert client.get(f"/batches/{uuid4()}/pre-label").status_code == 404 + + def test_a_point_prompt_connection_is_refused_before_a_job_exists( client: TestClient, in_annotation_batch: OpenBatch, segmenter_connection: str ) -> None: