From a26362df95d4f0ecab69be5a83cfe907de9b5125 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Wed, 12 Aug 2026 20:58:34 -0700 Subject: [PATCH 1/4] feat(inference): run local models on the Apple Silicon GPU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mps` joins the closed device vocabulary a local connection draws from, so a connection on an M-series Mac runs on the GPU instead of on the CPU cores. It was refused before because nothing in the adapters could honour it, which is the rule `DEVICE_PATTERN` states and which this keeps: `gpu` and `auto` still name nothing that could be resolved and stay out. Half precision remains CUDA-only. Metal has no float64 and its bfloat16 varies between releases, so `precisions_for` answers `fp32` alone for `mps` and the existing cross-field rule refuses the pairing at creation. The device-resolution rule was two identical private methods, one per adapter, and neither had a test — nothing reached the fallback branch, the warning or the half-precision decision. It is promoted into `inference/_device.py` and covered there, with availability injected so both answers are exercised on a machine that has neither GPU. `PYTORCH_ENABLE_MPS_FALLBACK` is set as `visionset.inference` is imported, not where the device is resolved: the array library reads it while it initialises, so by then it is already too late. It is a `setdefault`, so an operator who turned it off keeps that answer. `openapi.json` and the generated client are byte-identical — `device` travels as a plain string and no schema shape moved. --- docs/cli.md | 2 +- docs/inference.md | 61 +++++--- docs/install.md | 4 +- .../src/screens/inferenceCatalog.test.ts | 13 +- .../ui-core/src/screens/inferenceCatalog.ts | 20 ++- src/visionset/cli/inference.py | 12 +- src/visionset/inference/__init__.py | 16 ++ src/visionset/inference/_device.py | 104 +++++++++++++ src/visionset/inference/sam_provider.py | 25 +--- .../inference/transformers_provider.py | 40 +---- src/visionset/kernel/domain/__init__.py | 2 + src/visionset/kernel/domain/inference.py | 33 ++++- tests/architecture/test_optional_runtime.py | 27 ++++ tests/inference/stubs.py | 18 ++- tests/inference/test_device.py | 140 ++++++++++++++++++ tests/kernel/test_inference_connections.py | 26 +++- 16 files changed, 440 insertions(+), 103 deletions(-) create mode 100644 src/visionset/inference/_device.py create mode 100644 tests/inference/test_device.py diff --git a/docs/cli.md b/docs/cli.md index 8eb6a714..a9daefd5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -37,7 +37,7 @@ visionset token create --name NAME visionset token list visionset token revoke NAME [--yes] visionset inference create NAME --type local|http --model ID --revision REV - [--device cpu|cuda|cuda:N] [--precision fp16|fp32] + [--device cpu|mps|cuda|cuda:N] [--precision fp16|fp32] [--endpoint URL] visionset inference list visionset inference show|update|delete NAME_OR_ID diff --git a/docs/inference.md b/docs/inference.md index c0f776f5..aa53c351 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -35,7 +35,7 @@ with WorkspaceService.open("./road-signs") as workspace: model_id="some/model", model_revision="abc123", device="cuda", - precision="fp16", # fp16 needs a cuda device; a cpu connection is fp32 + precision="fp16", # fp16 needs a cuda device; cpu and mps connections are fp32 ) for one in connections.list(): print(one.name, one.connection_type.value, one.setup_state.value) @@ -209,24 +209,46 @@ nor the family, and neither does sending the same model reference back unchanged An `http` connection keeps no weights here, so a model edit resets nothing for it. It stays `ready`, which for that kind has always meant *there is nothing to set up on this machine*. -## Running on the CPU +## Which device runs the model -A connection asking for `cuda` on a machine with no GPU falls back to the CPU, in full precision, -with a warning in the log. It is a fallback rather than a preference - a workspace configured on a -workstation should still open on a laptop - but it is slower by a large factor, which is why it is -said out loud rather than silently done. +A local connection names the device it runs on, and there are three to name. -Half precision applies on CUDA only, and the kernel now says so rather than absorbing it: a `cpu` -connection asking for `fp16` is refused at creation. On a CPU it was never the conservative choice -it looks like - `float16` arithmetic outside CUDA's autocast is slower than the `float32` it was -avoiding - and a setting the adapters drop is one the row would otherwise go on displaying as -though it had an effect. - -**Both fields are closed vocabularies.** `device` is `cpu`, `cuda`, or `cuda:N` for the second GPU -on a machine that has one; `precision` is `fp16` or `fp32`, and `float16`, `half`, `float32` and -`full` are accepted as spellings of those two. Anything else is refused with a sentence naming the -members. What this closes is a gap rather than a freedom: `gpu` used to be accepted and then -resolved onto the CPU, so the connection described a run that never happened. +| Device | What it is | Precision | +| --- | --- | --- | +| `cpu` | The processor. Every machine has one, and it is the default a new connection opens on | `fp32` | +| `cuda` | An NVIDIA GPU. A machine with more than one addresses the rest as `cuda:1`, `cuda:2` and so on | `fp16` or `fp32` | +| `mps` | Apple Silicon's GPU, on an M-series Mac. There is only ever one of it | `fp32` | + +**On Apple Silicon nothing needs configuring beyond choosing the device.** The `local-inference` +extra is the same one everybody installs, the macOS wheels it brings carry Metal support already, +and there is no second package index, no environment variable and no build flag. Create the +connection with `mps` and it runs on the GPU. + +**A device this machine does not offer falls back to the CPU**, in full precision, with a warning +in the log naming the connection and the device it asked for. The same rule covers all three, so +`mps` on a machine with no Metal behaves exactly as `cuda` on a machine with no NVIDIA GPU. It is a +fallback rather than a preference - a workspace configured on a workstation should still open on a +laptop - but it is slower by a large factor, which is why it is said out loud rather than silently +done. + +**Half precision applies on CUDA only**, and the kernel says so rather than absorbing it: a `cpu` +or `mps` connection asking for `fp16` is refused at creation. On a CPU it was never the +conservative choice it looks like - `float16` arithmetic outside CUDA's autocast is slower than the +`float32` it was avoiding - and Metal has no float64 at all with a bfloat16 that varies between +releases, so full precision is the only format that behaves the same on every Mac. A setting the +adapters would drop is one the row would otherwise go on displaying as though it had an effect. + +Where Metal has no implementation for an operator a model reaches for, that one operator runs on +the CPU and the rest of the forward pass stays on the GPU. Nothing has to be turned on for this; +the adapters ask for it themselves. + +**Both fields are closed vocabularies.** `device` is `cpu`, `mps`, `cuda`, or `cuda:N` for the +second GPU on a machine that has one; `precision` is `fp16` or `fp32`, and `float16`, `half`, +`float32` and `full` are accepted as spellings of those two. Anything else is refused with a +sentence naming the members. What this closes is a gap rather than a freedom: `gpu` used to be +accepted and then resolved onto the CPU, so the connection described a run that never happened. +A device is in the vocabulary when the adapters can honour it, which is why `mps` is in it and +`gpu` and `auto` are not. ## What a connection can be asked for @@ -467,7 +489,8 @@ form. each one is pinned to a revision this build was checked against. **Custom model...** is the last entry and reveals the free model id and revision fields: the list guides, it does not restrict, and any model this build has an adapter for remains typeable. Device and precision are lists too, - and the precision list follows the device, because half precision applies on CUDA only. Underneath + and the precision list follows the device, because half precision applies on CUDA only - so + picking `mps` leaves `fp32` as the only precision offered. Underneath is what fetching that revision would cost - the size described above, read while you are still deciding. If this machine has no `local-inference` extra the size cannot be read, and the form says so, in the server's own words, with the install command. **It stays usable**: creating a @@ -516,7 +539,7 @@ downloaded, or nothing of the right kind - and each names a different thing to d visionset inference size some/model --revision abc123 visionset inference create local-detector \ --type local --model some/model --revision abc123 --device cuda --precision fp16 -# --device takes cpu, cuda or cuda:N; --precision takes fp16 or fp32, and fp16 needs a cuda device +# --device takes cpu, mps, cuda or cuda:N; --precision takes fp16 or fp32, and fp16 needs a cuda device visionset inference list visionset inference show local-detector --json visionset inference update local-detector --revision def456 diff --git a/docs/install.md b/docs/install.md index a8f43915..c8961893 100644 --- a/docs/install.md +++ b/docs/install.md @@ -79,7 +79,9 @@ pip install "visionset[local-inference]" ``` That brings torch, torchvision, transformers, accelerate and huggingface_hub - roughly two -gigabytes, most of it CUDA - which is exactly why it is not in the base install. Without it you can still create a +gigabytes, most of it CUDA - which is exactly why it is not in the base install. It is the same +command on every platform: the macOS wheels it installs carry Apple Silicon GPU support already, +so a Mac needs no second index and no build flag to run a connection on `mps`. Without it you can still create a local connection, list it, and see what it is configured for; what you cannot do is fetch its weights or ask it to predict. Both refusals name the command above rather than saying "unavailable", the way a missing `ffmpeg` does. diff --git a/frontend/ui-core/src/screens/inferenceCatalog.test.ts b/frontend/ui-core/src/screens/inferenceCatalog.test.ts index 32ba7ef2..93c5b71c 100644 --- a/frontend/ui-core/src/screens/inferenceCatalog.test.ts +++ b/frontend/ui-core/src/screens/inferenceCatalog.test.ts @@ -67,17 +67,24 @@ it("offers half precision on CUDA and on every address of it", () => { // A second GPU is still a GPU. This is the kernel's `precisions_for`, and the // two answer the same way or the form offers what the kernel refuses. expect(precisionsFor("cuda:1")).toEqual(["fp16", "fp32"]); + // Metal has no float64 and an inconsistent bfloat16, so full precision is the + // only format that behaves the same on every Mac — and the kernel refuses the + // pairing at creation, which a form still offering it would walk straight into. + expect(precisionsFor("mps")).toEqual(["fp32"]); }); it("keeps a precision that survives a device change and replaces one that does not", () => { expect(precisionOn("cuda", "fp32")).toBe("fp32"); expect(precisionOn("cpu", "fp32")).toBe("fp32"); expect(precisionOn("cpu", "fp16")).toBe("fp32"); + // Moving a half-precision CUDA connection onto Metal cannot keep the setting. + expect(precisionOn("mps", "fp16")).toBe("fp32"); }); -it("offers the two devices every machine can be asked about", () => { +it("offers the devices every machine can be asked about", () => { // `cuda:N` is deliberately absent: how many GPUs this machine has is not // something a static list can know, so it is typed by the kernel's pattern and - // shown by the form only when a row already carries one. - expect([...DEVICES]).toEqual(["cpu", "cuda"]); + // shown by the form only when a row already carries one. `mps` needs no such + // escape, because a Mac has exactly one. + expect([...DEVICES]).toEqual(["cpu", "cuda", "mps"]); }); diff --git a/frontend/ui-core/src/screens/inferenceCatalog.ts b/frontend/ui-core/src/screens/inferenceCatalog.ts index f045b57e..b177cf41 100644 --- a/frontend/ui-core/src/screens/inferenceCatalog.ts +++ b/frontend/ui-core/src/screens/inferenceCatalog.ts @@ -161,8 +161,14 @@ export function curatedEntry(modelId: string, revision: string): CuratedModel | return found !== undefined && found.revision === revision ? found : undefined; } -/** The devices a form offers, in the order it offers them. */ -export const DEVICES = ["cpu", "cuda"] as const; +/** + * The devices a form offers, in the order it offers them. + * + * `mps` is Apple Silicon's GPU, and it is one entry rather than a platform + * branch: which devices a machine actually has is answered where the model is + * loaded, not by a form guessing from a user agent. + */ +export const DEVICES = ["cpu", "cuda", "mps"] as const; /** * The precisions that are honoured on that device — the kernel's @@ -170,9 +176,13 @@ export const DEVICES = ["cpu", "cuda"] as const; * * Half precision is CUDA-only in both local adapters, so `cpu` + `fp16` is not a * slower run but a setting with no effect that the row would go on displaying as - * though it had one. A machine addressing a second GPU writes `cuda:1`, which is - * not a member of {@link DEVICES} and is still a CUDA device — hence the prefix - * test rather than an equality against `"cuda"`. + * though it had one. `mps` answers the same way and for its own reason: Metal has + * no float64 and an inconsistent bfloat16, so full precision is the only format + * that behaves the same on every machine offering the device. A machine + * addressing a second GPU writes `cuda:1`, which is not a member of + * {@link DEVICES} and is still a CUDA device — hence the prefix test rather than + * an equality against `"cuda"`, which is also what leaves every non-CUDA device + * on `fp32` without naming each one. */ export function precisionsFor(device: string): readonly Precision[] { return device.startsWith("cuda") ? ["fp16", "fp32"] : ["fp32"]; diff --git a/src/visionset/cli/inference.py b/src/visionset/cli/inference.py index ac30d4ab..ea6f2f50 100644 --- a/src/visionset/cli/inference.py +++ b/src/visionset/cli/inference.py @@ -76,11 +76,13 @@ def inference_create( ], device: Annotated[ str | None, - typer.Option("--device", help="Local only. cpu, cuda, or cuda:N for a second GPU."), + typer.Option("--device", help="Local only. cpu, mps, cuda, or cuda:N for a second GPU."), ] = None, precision: Annotated[ Precision | None, - typer.Option("--precision", help="Local only. fp16 needs a cuda device."), + typer.Option( + "--precision", help="Local only. fp16 needs a cuda device; cpu and mps run in fp32." + ), ] = None, endpoint_url: Annotated[ str | None, typer.Option("--endpoint", help="HTTP only. Where to send predictions.") @@ -134,11 +136,13 @@ def inference_update( model_revision: Annotated[str | None, typer.Option("--revision", help="Move the pin.")] = None, device: Annotated[ str | None, - typer.Option("--device", help="Local only. cpu, cuda, or cuda:N for a second GPU."), + typer.Option("--device", help="Local only. cpu, mps, cuda, or cuda:N for a second GPU."), ] = None, precision: Annotated[ Precision | None, - typer.Option("--precision", help="Local only. fp16 needs a cuda device."), + typer.Option( + "--precision", help="Local only. fp16 needs a cuda device; cpu and mps run in fp32." + ), ] = None, endpoint_url: Annotated[str | None, typer.Option("--endpoint", help="HTTP only.")] = None, json_out: JsonOption = False, diff --git a/src/visionset/inference/__init__.py b/src/visionset/inference/__init__.py index 8a3bde5b..cb0a0bee 100644 --- a/src/visionset/inference/__init__.py +++ b/src/visionset/inference/__init__.py @@ -31,6 +31,17 @@ connection may hold a detector that answers words or a segmenter that answers places and those are not interchangeable. +**One environment variable is set as this module is read**, and it is the only +side effect importing this package has. ``PYTORCH_ENABLE_MPS_FALLBACK`` is what +lets an operator Metal has not implemented run on the CPU instead of raising, and +the array library reads it while it initialises rather than when such an operator +is reached — so by the time a connection has been resolved to a device it is +already too late to set. This module is the earliest place that is certain to be +read before torch is imported anywhere in the package, which is what makes it the +right place despite the setting having nothing to do with composition. It costs +one dictionary write on every machine, does nothing at all on a machine with no +Metal, and ``setdefault`` leaves an operator who set it to ``0`` alone. + **What each surface reaches for.** ``fetch_weights`` is the download, ``check_integrity`` is the full re-read that tells damage from completeness, ``suggest`` is one click's worth of interactive segmentation, and ``provider_for`` @@ -41,6 +52,7 @@ from __future__ import annotations +from visionset.inference._device import MPS_FALLBACK_VARIABLE, enable_mps_fallback from visionset.inference._extra import EXTRA, INSTALL_COMMAND, MODULES, require from visionset.inference.cache import ( DEFAULT_EMBEDDING_CAPACITY, @@ -97,8 +109,12 @@ with_families, ) +enable_mps_fallback() + __all__ = [ "EPSILON", + "MPS_FALLBACK_VARIABLE", + "enable_mps_fallback", "DEFAULT_EMBEDDING_CAPACITY", "DEFAULT_IOU_THRESHOLD", "DEFAULT_PROVIDER_CAPACITY", diff --git a/src/visionset/inference/_device.py b/src/visionset/inference/_device.py new file mode 100644 index 00000000..a1e25c23 --- /dev/null +++ b/src/visionset/inference/_device.py @@ -0,0 +1,104 @@ +# usage: from visionset.inference._device import resolved +"""Where a local model actually runs, and whether half precision survives the trip. + +**One module because it was two identical methods.** Both local adapters carried +a private ``_resolved_device`` and a ``CPU_FALLBACK_WARNING`` that were the same +text, and adding a third device to two copies is how two copies become two +answers. The rule is promoted here rather than duplicated a third time, which is +the same move ``require_move`` and ``require_draft`` made in the kernel. + +**A run-time question, not a configuration one.** What the connection holds is a +*request*, written on a machine that may not be this one; the vocabulary it is +drawn from lives in the kernel, and whether a named device is present right now +lives here. The two are deliberately separate: a device nothing could ever +address is refused when it is written down, and a device this particular machine +does not have is answered at the moment of the call. + +**Falling back is the answer, not refusing.** A connection asking for a device +this machine does not offer runs on the CPU and says so at WARNING. Refusing +would make a workspace configured on a workstation unusable on a laptop, which is +worse than being slow; staying silent would make it fifty times slower for no +visible reason, which is worse than being loud. The same rule covers every +device, so ``mps`` on a machine without Metal behaves exactly as ``cuda`` on a +machine without an NVIDIA GPU. + +**``torch`` is passed in rather than imported**, for the reason the whole package +defers its heavy imports — and with the side benefit that a stub proves every +branch of this on a machine that has neither GPU. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any, Final + +from visionset.inference import _fp16 +from visionset.kernel.domain import CPU, CUDA, MPS + +_logger: Final = logging.getLogger(__name__) + +CPU_FALLBACK_WARNING: Final = ( + "inference connection %r asks for device %r, which this machine does not offer; " + "running on the CPU in full precision instead" +) +"""Said out loud, once, at WARNING. A fallback that happens silently is a +fifty-times-slower run somebody spends an afternoon not understanding — the +~115 ms an image the detector is measured at is a GPU figure.""" + +MPS_FALLBACK_VARIABLE: Final = "PYTORCH_ENABLE_MPS_FALLBACK" +"""What lets an operator Metal has not implemented run on the CPU instead of raising. + +Set to ``1`` in :mod:`visionset.inference`'s own module body, because the array +library reads it while it initialises rather than when an operator is reached, so +by the time a device has been resolved it is already too late to set. It is set +here as well for a caller that reached an adapter without importing the package +around it, and both are :func:`os.environ.setdefault`, so an operator who set it +to ``0`` deliberately keeps that answer. +""" + + +def enable_mps_fallback() -> None: + """Ask for unimplemented operators to run on the CPU rather than raise. + + Free on every machine that has no Metal at all, which is why it is set + unconditionally at import and needs no availability check of its own. + """ + os.environ.setdefault(MPS_FALLBACK_VARIABLE, "1") + + +def resolved( + torch: Any, *, device: str, precision: str | None, connection_name: str +) -> tuple[str, bool]: + """The device this will really run on, and whether it runs in half precision. + + Half precision is CUDA-only, and the test is on the device that *survived* + rather than on the one that was asked for. fp16 outside CUDA is not the + conservative choice it looks like: the shims in :mod:`._fp16` exist for + CUDA's autocast, Metal has no float64 and an inconsistent bfloat16, and + ``float16`` arithmetic on a CPU is slower than the float32 it was avoiding. + The kernel's ``precisions_for`` refuses the pairing before it is ever stored; + this is the same rule holding for a row written before it did. + """ + wanted = device.strip() + if not _present(torch, wanted): + _logger.warning(CPU_FALLBACK_WARNING, connection_name, wanted) + return CPU, False + if wanted == MPS: + enable_mps_fallback() + return wanted, wanted.startswith(CUDA) and _fp16.wants_half(precision) + + +def _present(torch: Any, device: str) -> bool: + """Whether this machine offers that device, right now. + + ``mps`` is asked with ``is_available`` alone. ``is_built`` answers a + different question — whether this build of the array library carries the + backend — and ``is_available`` is already false when it does not, so asking + both is asking one question twice. + """ + if device.startswith(CUDA): + return bool(torch.cuda.is_available()) + if device == MPS: + return bool(torch.backends.mps.is_available()) + return True diff --git a/src/visionset/inference/sam_provider.py b/src/visionset/inference/sam_provider.py index a68b6c40..afdfd28c 100644 --- a/src/visionset/inference/sam_provider.py +++ b/src/visionset/inference/sam_provider.py @@ -37,7 +37,6 @@ from __future__ import annotations -import logging from collections.abc import Iterator from io import BytesIO from pathlib import Path @@ -46,7 +45,7 @@ from PIL import Image -from visionset.inference import _fp16 +from visionset.inference import _device, _fp16 from visionset.inference._extra import imported from visionset.inference.cache import DEFAULT_EMBEDDING_CAPACITY, BoundedCache from visionset.kernel.domain import ( @@ -58,8 +57,6 @@ ) from visionset.kernel.errors import UnsupportedPrompt -_logger: Final = logging.getLogger(__name__) - POSITIVE: Final = 1 NEGATIVE: Final = 0 """What this family calls a point that says *this* and one that says *not that*. @@ -70,11 +67,6 @@ is exactly the kind of thing that lives in an adapter. """ -CPU_FALLBACK_WARNING: Final = ( - "inference connection %r asks for device %r, which this machine does not offer; " - "running on the CPU in full precision instead" -) - def points_and_labels(prompt: PointPrompt) -> tuple[list[list[float]], list[int]]: """The prompt as this family wants it: one flat point list, one label list. @@ -286,7 +278,12 @@ def _ready(self) -> tuple[Any, Any, str, bool]: def _load(self) -> tuple[Any, Any, str, bool]: torch = imported("torch") transformers = imported("transformers") - device, half = self._resolved_device(torch) + device, half = _device.resolved( + torch, + device=self._device, + precision=self._precision, + connection_name=self._connection_name, + ) common = { "revision": self._model_revision, "cache_dir": str(self._cache_dir), @@ -299,11 +296,3 @@ def _load(self) -> tuple[Any, Any, str, bool]: **common, ) return processor, model.to(device).eval(), device, half - - def _resolved_device(self, torch: Any) -> tuple[str, bool]: - """Where this runs, and whether half precision survives — the sibling's rule.""" - wanted = self._device.strip() - if wanted.startswith("cuda") and not torch.cuda.is_available(): - _logger.warning(CPU_FALLBACK_WARNING, self._connection_name, wanted) - return "cpu", False - return wanted, wanted.startswith("cuda") and _fp16.wants_half(self._precision) diff --git a/src/visionset/inference/transformers_provider.py b/src/visionset/inference/transformers_provider.py index a1ce625d..445b2596 100644 --- a/src/visionset/inference/transformers_provider.py +++ b/src/visionset/inference/transformers_provider.py @@ -25,7 +25,6 @@ from __future__ import annotations -import logging from collections.abc import Iterator, Sequence from io import BytesIO from pathlib import Path @@ -33,7 +32,7 @@ from PIL import Image -from visionset.inference import _fp16 +from visionset.inference import _device, _fp16 from visionset.inference._extra import imported from visionset.inference.nms import DEFAULT_IOU_THRESHOLD, suppressed from visionset.kernel.domain import ( @@ -46,8 +45,6 @@ ) from visionset.kernel.errors import UnsupportedPrompt -_logger: Final = logging.getLogger(__name__) - DEFAULT_TEXT_THRESHOLD: Final = 0.25 """How sure the model must be that a box matches a *phrase*, as opposed to that it is an object at all. @@ -59,14 +56,6 @@ used. """ -CPU_FALLBACK_WARNING: Final = ( - "inference connection %r asks for device %r, which this machine does not offer; " - "running on the CPU in full precision instead" -) -"""Said out loud, once, at WARNING. A fallback that happens silently is a -fifty-times-slower run somebody spends an afternoon not understanding — the -~115 ms an image this adapter is measured at is a GPU figure.""" - def regions_from( boxes: Sequence[Sequence[float]], @@ -271,7 +260,12 @@ def _ready(self) -> tuple[Any, Any, str, bool]: def _load(self) -> tuple[Any, Any, str, bool]: torch = imported("torch") transformers = imported("transformers") - device, half = self._resolved_device(torch) + device, half = _device.resolved( + torch, + device=self._device, + precision=self._precision, + connection_name=self._connection_name, + ) common = { "revision": self._model_revision, "cache_dir": str(self._cache_dir), @@ -286,26 +280,6 @@ def _load(self) -> tuple[Any, Any, str, bool]: ) return processor, model.to(device).eval(), device, half - def _resolved_device(self, torch: Any) -> tuple[str, bool]: - """Where this actually runs, and whether half precision survives the trip. - - The CPU fallback is a fallback, not a preference: a connection asking for - ``cuda`` on a machine with none gets the CPU **and a warning**, because - the alternative — refusing — would make a workspace configured on a - laptop unusable on it, and the alternative to the warning is a run that - is fifty times slower for no visible reason. - - Falling back also drops half precision. fp16 on a CPU is not the - conservative choice it looks like: the shims above exist for CUDA's - autocast, and ``float16`` arithmetic outside it is slower than the - float32 it was avoiding. - """ - wanted = self._device.strip() - if wanted.startswith("cuda") and not torch.cuda.is_available(): - _logger.warning(CPU_FALLBACK_WARNING, self._connection_name, wanted) - return "cpu", False - return wanted, wanted.startswith("cuda") and _fp16.wants_half(self._precision) - def _labels_in(raw: dict[str, Any]) -> list[str]: """The phrase each box answered under. diff --git a/src/visionset/kernel/domain/__init__.py b/src/visionset/kernel/domain/__init__.py index eed0e314..e1912ef1 100644 --- a/src/visionset/kernel/domain/__init__.py +++ b/src/visionset/kernel/domain/__init__.py @@ -74,6 +74,7 @@ EVERY_CONNECTION_TYPE, EVERY_SETUP_STATE, INTEGRITY_CHECK_JOB_TYPE, + MPS, OFFERED_DEVICES, WEIGHT_DOWNLOAD_JOB_TYPE, WEIGHT_HOLDING_TYPES, @@ -249,6 +250,7 @@ "CONNECTION_JOB_KEY", "CPU", "CUDA", + "MPS", "DEVICE_PATTERN", "OFFERED_DEVICES", "precisions_for", diff --git a/src/visionset/kernel/domain/inference.py b/src/visionset/kernel/domain/inference.py index 8b7be77a..0b5cc161 100644 --- a/src/visionset/kernel/domain/inference.py +++ b/src/visionset/kernel/domain/inference.py @@ -148,7 +148,15 @@ class Precision(StrEnum): CUDA: Final = "cuda" """The default GPU. A machine with several addresses the rest as ``cuda:1``, ``cuda:2``…""" -OFFERED_DEVICES: Final[tuple[str, ...]] = (CPU, CUDA) +MPS: Final = "mps" +"""Apple Silicon's GPU, and there is only ever one of it. + +Named for the framework that drives it rather than for the hardware, which is +how the array library spells it and therefore the only spelling an adapter can +hand on unchanged. +""" + +OFFERED_DEVICES: Final[tuple[str, ...]] = (CPU, CUDA, MPS) """The devices a form offers, in the order it offers them. Not the whole of what :data:`DEVICE_PATTERN` accepts, and the difference is @@ -158,16 +166,20 @@ class Precision(StrEnum): than silently rewriting it to the nearest member. """ -DEVICE_PATTERN: Final = re.compile(r"^(?:cpu|cuda(?::\d+)?)$") +DEVICE_PATTERN: Final = re.compile(r"^(?:cpu|mps|cuda(?::\d+)?)$") """Every device string this build can honestly run on. A pattern rather than an enum because of the one member that is not a fixed -word. What is *not* here is the point: ``gpu``, ``mps``, ``auto`` and every +word. The rule is that a device is here when this build can *honour* it, and +what is not here is as much the point as what is: ``gpu``, ``auto`` and every typo were accepted before and then quietly fell back to the CPU in full -precision — a connection that names a runtime it never gets. The adapters still -fall back when a *valid* device turns out to be absent at run time, which is a -fact about the machine at the moment of the call and belongs there; a device -nothing could ever address is a fact about the configuration and belongs here. +precision — a connection that names a runtime it never gets. ``mps`` was out for +that same reason and is in now, because the adapters resolve it, run on it, and +condition its precision like any other device rather than degrading it in +silence. The adapters still fall back when a *valid* device turns out to be +absent at run time, which is a fact about the machine at the moment of the call +and belongs there; a device nothing could ever address is a fact about the +configuration and belongs here. """ @@ -181,11 +193,16 @@ def precisions_for(device: str) -> tuple[Precision, ...]: setting that has no effect at all — and one the row would go on displaying as though it did. + ``mps`` answers the same way ``cpu`` does, and for a reason of its own rather + than by inheriting the adapters' rule: Metal has no float64 at all and its + bfloat16 is inconsistent across releases, so full precision is the only + numeric format that behaves the same on every machine that offers the device. + Takes the string rather than a member because ``cuda:1`` is a device and not an enum, and returns a tuple rather than a set because a caller offering a choice needs an order and a caller checking membership does not care. """ - return (Precision.FP32,) if device == CPU else (Precision.FP16, Precision.FP32) + return (Precision.FP32,) if device in (CPU, MPS) else (Precision.FP16, Precision.FP32) EVERY_CONNECTION_TYPE: Final[frozenset[ConnectionType]] = frozenset(ConnectionType) diff --git a/tests/architecture/test_optional_runtime.py b/tests/architecture/test_optional_runtime.py index ae6d2202..c0977f9a 100644 --- a/tests/architecture/test_optional_runtime.py +++ b/tests/architecture/test_optional_runtime.py @@ -63,6 +63,33 @@ def test_a_base_install_imports_none_of_the_optional_runtime() -> None: assert result.returncode == 0, result.stderr +def test_importing_the_inference_package_asks_for_the_metal_cpu_fallback() -> None: + """And does so *before* the array library could have been imported. + + ``PYTORCH_ENABLE_MPS_FALLBACK`` is read while torch initialises rather than + when an unimplemented operator is reached, so setting it after a device has + been resolved would be too late. The claim worth making is therefore about + ordering, and the only interpreter that can answer it is one that has + imported nothing else — in this one, the suite has already imported both. + + The probe deletes the variable first: a developer who exports it would + otherwise be told the code sets it when nothing did. + """ + probe = f""" +import os +import sys + +os.environ.pop("PYTORCH_ENABLE_MPS_FALLBACK", None) +import visionset.inference + +assert os.environ.get("PYTORCH_ENABLE_MPS_FALLBACK") == "1", "the fallback was not asked for" +loaded = {set(MODULES)} & set(sys.modules) +assert not loaded, f"the fallback was set after the runtime loaded: {{loaded}}" +""" + result = subprocess.run([sys.executable, "-c", probe], capture_output=True, text=True) + assert result.returncode == 0, result.stderr + + def test_creating_the_application_loads_none_of_it_either() -> None: """Importing is not the whole of startup — `create_app()` runs too. diff --git a/tests/inference/stubs.py b/tests/inference/stubs.py index b64d7a3d..ef5f5a08 100644 --- a/tests/inference/stubs.py +++ b/tests/inference/stubs.py @@ -150,12 +150,21 @@ def __exit__(self, *_: Any) -> None: class StubTorch: - """Enough of torch for ``forward_guard`` to do its work and restore itself.""" + """Enough of torch for ``forward_guard`` and ``_device.resolved`` to work. + + The two availability answers are constructor arguments rather than fixed + values, because device resolution is a *branch* on them and a stub that can + only say "no GPU here" can only ever exercise the fallback. Both default to + absent, which is what every machine running this suite actually is and what + every caller predating the arguments expects. + """ float16 = "float16" - def __init__(self) -> None: + def __init__(self, *, cuda: bool = False, mps: bool = False) -> None: self.nn = SimpleNamespace(functional=Functional()) + self.cuda = SimpleNamespace(is_available=lambda: cuda) + self.backends = SimpleNamespace(mps=SimpleNamespace(is_available=lambda: mps)) def no_grad(self) -> _Scope: return _Scope() @@ -163,11 +172,6 @@ def no_grad(self) -> _Scope: def autocast(self, device_type: str, dtype: str) -> _Scope: return _Scope() - class cuda: # noqa: N801 — mirrors torch's own spelling - @staticmethod - def is_available() -> bool: - return False - def disc(radius: int, *, size: int = 64) -> Mask: """A filled circle — a mask with an outline worth simplifying.""" diff --git a/tests/inference/test_device.py b/tests/inference/test_device.py new file mode 100644 index 00000000..45e4f9f0 --- /dev/null +++ b/tests/inference/test_device.py @@ -0,0 +1,140 @@ +"""Where a local model runs, and whether half precision survives the trip. + +The rule these cover shipped inside both adapters, twice, and untested in either: +nothing reached the fallback branch, the warning, or the half-precision decision. +It is one function now, and this is its first coverage. + +**Every case runs on a stub**, which is the point rather than a compromise. No +runner in continuous integration has an NVIDIA GPU and none is an Apple Silicon +Mac, so a test asking for real hardware would skip everywhere and prove nothing; +availability is an *input* to this function, and injecting it is how both answers +get exercised on a machine that has neither. +""" + +from __future__ import annotations + +import logging +import os + +import pytest +from tests.inference.stubs import StubTorch + +from visionset.inference import _device + + +def resolve( + *, device: str, precision: str | None = "fp32", cuda: bool = False, mps: bool = False +) -> tuple[str, bool]: + """``resolved`` with a stub torch and a name, so a case reads as its question.""" + return _device.resolved( + StubTorch(cuda=cuda, mps=mps), + device=device, + precision=precision, + connection_name="detector", + ) + + +def test_the_cpu_is_always_present_and_never_runs_in_half_precision() -> None: + assert resolve(device="cpu") == ("cpu", False) + + +def test_a_cuda_machine_keeps_cuda_and_the_half_precision_it_asked_for() -> None: + assert resolve(device="cuda", precision="fp16", cuda=True) == ("cuda", True) + + +def test_a_cuda_connection_asking_for_full_precision_keeps_full_precision() -> None: + assert resolve(device="cuda", precision="fp32", cuda=True) == ("cuda", False) + + +def test_a_second_gpu_is_still_a_cuda_device() -> None: + assert resolve(device="cuda:1", precision="fp16", cuda=True) == ("cuda:1", True) + + +def test_cuda_on_a_machine_with_no_gpu_falls_back_to_the_cpu( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING): + assert resolve(device="cuda", precision="fp16", cuda=False) == ("cpu", False) + assert "detector" in caplog.text + assert "cuda" in caplog.text + + +def test_mps_on_an_apple_silicon_machine_is_kept() -> None: + assert resolve(device="mps", mps=True) == ("mps", False) + + +def test_mps_never_runs_in_half_precision_even_when_the_connection_asks_for_it() -> None: + """The CUDA-only dtype rule must not leak to Metal. + + A row written before the kernel conditioned precision on the device could + carry ``mps`` beside ``fp16``, and the answer has to be the same one the + kernel gives at creation rather than an autocast Metal cannot honour. + """ + assert resolve(device="mps", precision="fp16", mps=True) == ("mps", False) + + +def test_a_cpu_connection_asking_for_half_precision_does_not_get_it() -> None: + assert resolve(device="cpu", precision="fp16") == ("cpu", False) + + +def test_mps_on_a_machine_without_metal_falls_back_to_the_cpu( + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.WARNING): + assert resolve(device="mps", mps=False) == ("cpu", False) + assert "detector" in caplog.text + assert "mps" in caplog.text + + +def test_the_cpu_is_not_asked_whether_it_is_available( + caplog: pytest.LogCaptureFixture, +) -> None: + """A stub answering ``False`` to everything must still leave the CPU alone. + + Without this, a resolver that asked about every device would look correct in + every other case here and warn on the one machine that cannot be wrong. + """ + with caplog.at_level(logging.WARNING): + assert resolve(device="cpu") == ("cpu", False) + assert caplog.text == "" + + +def test_surrounding_whitespace_in_a_stored_device_is_ignored() -> None: + assert resolve(device=" cuda ", precision="fp16", cuda=True) == ("cuda", True) + + +def test_resolving_onto_metal_asks_for_the_cpu_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv(_device.MPS_FALLBACK_VARIABLE, raising=False) + resolve(device="mps", mps=True) + assert os.environ[_device.MPS_FALLBACK_VARIABLE] == "1" + + +def test_an_operator_who_turned_the_fallback_off_keeps_that_answer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """``setdefault``, not an assignment. + + Somebody debugging which operators Metal is missing turns this off on + purpose, and a library that overwrites it makes that impossible. + """ + monkeypatch.setenv(_device.MPS_FALLBACK_VARIABLE, "0") + resolve(device="mps", mps=True) + assert os.environ[_device.MPS_FALLBACK_VARIABLE] == "0" + + +def test_the_fallback_can_be_asked_for_without_resolving_anything( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The call the package makes as it is imported, which is the one that matters. + + By the time a device has been resolved the array library is already loaded, + so the ``setdefault`` inside ``resolved`` is a second belt. That the *import* + reaches this before anything heavy loads is asserted in a fresh interpreter, + in ``tests/architecture/test_optional_runtime.py``, because this one has + already imported both. + """ + monkeypatch.delenv(_device.MPS_FALLBACK_VARIABLE, raising=False) + _device.enable_mps_fallback() + assert os.environ[_device.MPS_FALLBACK_VARIABLE] == "1" diff --git a/tests/kernel/test_inference_connections.py b/tests/kernel/test_inference_connections.py index 12c538e8..647f1acc 100644 --- a/tests/kernel/test_inference_connections.py +++ b/tests/kernel/test_inference_connections.py @@ -28,6 +28,7 @@ from visionset.kernel.domain import ( CPU, CUDA, + MPS, ConnectionSetupState, ConnectionType, InferenceConnection, @@ -133,7 +134,14 @@ def test_the_service_refuses_in_the_kernels_own_vocabulary(connections) -> None: @pytest.mark.parametrize( ("written", "stored"), - [("cpu", "cpu"), ("cuda", "cuda"), ("cuda:1", "cuda:1"), (" CUDA ", "cuda")], + [ + ("cpu", "cpu"), + ("cuda", "cuda"), + ("cuda:1", "cuda:1"), + (" CUDA ", "cuda"), + ("mps", "mps"), + (" MPS ", "mps"), + ], ) def test_a_device_this_build_can_address_is_kept_and_normalized(written: str, stored: str) -> None: """Case and surrounding space are forgiven; `cuda:N` is a device, not a typo.""" @@ -141,13 +149,19 @@ def test_a_device_this_build_can_address_is_kept_and_normalized(written: str, st assert made.device == stored -@pytest.mark.parametrize("written", ["gpu", "mps", "cuda:", "cuda:x", "cuda 1", "", "cpu0"]) +@pytest.mark.parametrize( + "written", ["gpu", "auto", "cuda:", "cuda:x", "cuda 1", "", "cpu0", "mps:0"] +) def test_a_device_nothing_here_could_address_is_refused(written: str) -> None: """The gap this closes: every one of these was accepted and then ignored. - The adapters resolve anything that is not CUDA onto the CPU in full + The adapters resolve anything they cannot honour onto the CPU in full precision, so a connection saying `gpu` used to describe a run that never - happened and went on displaying `gpu` while it did not happen. + happened and went on displaying `gpu` while it did not happen. `mps` used to + be refused for that same reason and is a device now, because the adapters + resolve it; `gpu` and `auto` still name nothing they could resolve. There is + only ever one Metal GPU, so `mps:0` is a typo rather than the second-device + escape `cuda:N` is. """ with pytest.raises(ValidationError, match="not a device this build can run on"): InferenceConnection(name="x", **(dict(LOCAL) | {"device": written})) @@ -196,6 +210,9 @@ def test_half_precision_is_refused_on_a_cpu_and_offered_on_a_gpu() -> None: with pytest.raises(ValidationError, match="fp16 is not available on cpu"): InferenceConnection(name="x", **(dict(LOCAL) | {"device": "cpu", "precision": "fp16"})) + with pytest.raises(ValidationError, match="fp16 is not available on mps"): + InferenceConnection(name="x", **(dict(LOCAL) | {"device": "mps", "precision": "fp16"})) + for device in ("cuda", "cuda:1"): made = InferenceConnection( name="x", **(dict(LOCAL) | {"device": device, "precision": "fp16"}) @@ -210,6 +227,7 @@ def test_the_conditioning_rule_has_one_owner() -> None: refuses, which is the shape `ui-capabilities` bans one layer up. """ assert precisions_for(CPU) == (Precision.FP32,) + assert precisions_for(MPS) == (Precision.FP32,) assert precisions_for(CUDA) == (Precision.FP16, Precision.FP32) assert precisions_for("cuda:3") == (Precision.FP16, Precision.FP32) From a29393f0147544e0e1909e5703c5652ff930a4d5 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Fri, 14 Aug 2026 02:10:27 -0700 Subject: [PATCH 2/4] fix(inference): Metal being available is not Metal being usable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `is_available` answers 'is there a Metal device', which is not 'can I put a tensor on it'. On an Intel Mac with a discrete GPU it answers true and every allocation then raises 'MPS backend is only supported on devices with unified memory' — so `_present` waved `mps` through, the documented CPU fallback never ran, no warning was logged, and the failure surfaced as a 500 out of the first suggestion. Asking `is_built` as well would not have helped; it is true there too. The second question is asked by doing it: one cached one-element allocation, which is the run-time shape this module already argues for. Reproduced and fixed against an i9 MacBook Pro, macOS 26.6, torch 2.13. cf. #573 --- src/visionset/inference/_device.py | 42 +++++++++++++++++++++--- tests/inference/stubs.py | 32 ++++++++++++++---- tests/inference/test_device.py | 52 ++++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 14 deletions(-) diff --git a/src/visionset/inference/_device.py b/src/visionset/inference/_device.py index a1e25c23..4958628d 100644 --- a/src/visionset/inference/_device.py +++ b/src/visionset/inference/_device.py @@ -31,6 +31,7 @@ import logging import os +from functools import lru_cache from typing import Any, Final from visionset.inference import _fp16 @@ -92,13 +93,44 @@ def resolved( def _present(torch: Any, device: str) -> bool: """Whether this machine offers that device, right now. - ``mps`` is asked with ``is_available`` alone. ``is_built`` answers a - different question — whether this build of the array library carries the - backend — and ``is_available`` is already false when it does not, so asking - both is asking one question twice. + ``mps`` is asked twice, and the second question is the one that matters. + ``is_available`` answers *is there a Metal device*, which is not the same as + *can I put a tensor on it*: on an Intel Mac with a discrete GPU it answers + true and every allocation then raises ``MPS backend is only supported on + devices with unified memory``. Measured on an i9 MacBook Pro, macOS 26.6, + torch 2.13 — ``is_built`` and ``is_available`` both true, ``torch.zeros(1, + device="mps")`` fatal. Asking ``is_built`` as well would not have helped; it + is true there too. + + So the second question is asked by *doing it*, which is the only form that + cannot be wrong, and it is the shape this module already argues for: a + run-time question answered at the moment of the call. """ if device.startswith(CUDA): return bool(torch.cuda.is_available()) if device == MPS: - return bool(torch.backends.mps.is_available()) + return bool(torch.backends.mps.is_available()) and _mps_serves(torch) + return True + + +@lru_cache(maxsize=None) +def _mps_serves(torch: Any) -> bool: + """Whether Metal will really take a tensor, asked by handing it one. + + **Cached on the array library itself**, so the cost is one allocation of one + element per process rather than per suggestion — and so that a test handing + in a different stub gets a different answer without having to clear + anything, which is what keeps this probe as injectable as the two + availability flags beside it. + + Only ``RuntimeError`` is caught, because that is what an unusable backend + raises. Anything else — a stub missing ``zeros``, an import that half + happened — is a surprise this function has no business converting into a + quiet "no GPU here", which is the most expensive kind of wrong answer a + fallback can give. + """ + try: + torch.zeros(1, device=MPS) + except RuntimeError: + return False return True diff --git a/tests/inference/stubs.py b/tests/inference/stubs.py index ef5f5a08..88d352db 100644 --- a/tests/inference/stubs.py +++ b/tests/inference/stubs.py @@ -152,19 +152,39 @@ def __exit__(self, *_: Any) -> None: class StubTorch: """Enough of torch for ``forward_guard`` and ``_device.resolved`` to work. - The two availability answers are constructor arguments rather than fixed - values, because device resolution is a *branch* on them and a stub that can - only say "no GPU here" can only ever exercise the fallback. Both default to - absent, which is what every machine running this suite actually is and what - every caller predating the arguments expects. + The availability answers are constructor arguments rather than fixed values, + because device resolution is a *branch* on them and a stub that can only say + "no GPU here" can only ever exercise the fallback. Both default to absent, + which is what every machine running this suite actually is and what every + caller predating the arguments expects. + + ``mps_usable`` is the third, and it is a separate axis from ``mps`` because + on a real machine the two genuinely disagree: an Intel Mac with a discrete + GPU reports Metal available and then refuses every allocation on it. It + defaults to true so that ``mps=True`` alone still means a working Apple + Silicon GPU, which is what every case written before this existed meant. """ float16 = "float16" - def __init__(self, *, cuda: bool = False, mps: bool = False) -> None: + def __init__( + self, *, cuda: bool = False, mps: bool = False, mps_usable: bool = True + ) -> None: self.nn = SimpleNamespace(functional=Functional()) self.cuda = SimpleNamespace(is_available=lambda: cuda) self.backends = SimpleNamespace(mps=SimpleNamespace(is_available=lambda: mps)) + self._mps_usable = mps_usable + + def zeros(self, *_: Any, device: str | None = None) -> Any: + """The probe's allocation, and the only tensor this stub ever makes. + + Raises what torch raises, with the message torch uses, because the + production code catches ``RuntimeError`` specifically and a stub raising + anything else would let that narrowing pass a test it should fail. + """ + if device == "mps" and not self._mps_usable: + raise RuntimeError("MPS backend is only supported on devices with unified memory") + return SimpleNamespace() def no_grad(self) -> _Scope: return _Scope() diff --git a/tests/inference/test_device.py b/tests/inference/test_device.py index 45e4f9f0..39ac1981 100644 --- a/tests/inference/test_device.py +++ b/tests/inference/test_device.py @@ -23,11 +23,21 @@ def resolve( - *, device: str, precision: str | None = "fp32", cuda: bool = False, mps: bool = False + *, + device: str, + precision: str | None = "fp32", + cuda: bool = False, + mps: bool = False, + mps_usable: bool = True, ) -> tuple[str, bool]: - """``resolved`` with a stub torch and a name, so a case reads as its question.""" + """``resolved`` with a stub torch and a name, so a case reads as its question. + + A fresh ``StubTorch`` per call is what keeps ``_mps_serves``'s cache out of + the way: it is keyed on the array library it was handed, so two cases asking + opposite questions never see each other's answer. + """ return _device.resolved( - StubTorch(cuda=cuda, mps=mps), + StubTorch(cuda=cuda, mps=mps, mps_usable=mps_usable), device=device, precision=precision, connection_name="detector", @@ -77,6 +87,42 @@ def test_a_cpu_connection_asking_for_half_precision_does_not_get_it() -> None: assert resolve(device="cpu", precision="fp16") == ("cpu", False) +def test_metal_that_cannot_take_a_tensor_falls_back_to_the_cpu( + caplog: pytest.LogCaptureFixture, +) -> None: + """The case ``is_available`` alone gets wrong, and it is a real machine. + + An Intel Mac with a discrete GPU answers ``is_built`` **and** + ``is_available`` true, then raises on every allocation — so the check that + shipped waved ``mps`` through, no warning was logged, and the failure landed + as a 500 out of the first suggestion instead of a slow run on the CPU. + Reproduced on an i9 MacBook Pro, macOS 26.6, torch 2.13. + """ + with caplog.at_level(logging.WARNING): + assert resolve(device="mps", mps=True, mps_usable=False) == ("cpu", False) + assert "detector" in caplog.text + assert "mps" in caplog.text + + +def test_the_probe_is_not_paid_for_on_a_machine_with_no_metal_at_all() -> None: + """``is_available`` is asked first, so the common case allocates nothing. + + A stub whose ``zeros`` raises ``AssertionError`` — which the production code + deliberately does *not* catch — is how "never reached" is asserted rather + than described. + """ + torch = StubTorch(mps=False) + torch.zeros = _never # type: ignore[method-assign] + + assert _device.resolved( + torch, device="mps", precision="fp32", connection_name="detector" + ) == ("cpu", False) + + +def _never(*_: object, **__: object) -> object: + raise AssertionError("the probe ran on a machine that reports no Metal") + + def test_mps_on_a_machine_without_metal_falls_back_to_the_cpu( caplog: pytest.LogCaptureFixture, ) -> None: From 6fef7c8e59a638dab9b38d170af5f5964fe5cd5f Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Fri, 14 Aug 2026 02:11:04 -0700 Subject: [PATCH 3/4] test(inference): the probe's RuntimeError narrowing had no coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation widening it to `except Exception` came back green — the docstring's claim that a non-RuntimeError surprise propagates rather than being read as absent hardware was a description, not a rule. cf. #573 --- tests/inference/test_device.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/inference/test_device.py b/tests/inference/test_device.py index 39ac1981..38f7451e 100644 --- a/tests/inference/test_device.py +++ b/tests/inference/test_device.py @@ -123,6 +123,25 @@ def _never(*_: object, **__: object) -> object: raise AssertionError("the probe ran on a machine that reports no Metal") +def test_a_surprise_from_the_probe_is_not_read_as_absent_hardware() -> None: + """The narrowing to ``RuntimeError``, which is the whole of what it buys. + + An unusable backend raises ``RuntimeError``; a half-installed array library + raises other things. Turning the second into a silent CPU fallback would + hide a broken installation behind a fifty-times-slower run, which is the + most expensive way for this function to be wrong — so it propagates. + """ + torch = StubTorch(mps=True) + torch.zeros = _broken # type: ignore[method-assign] + + with pytest.raises(ValueError, match="half an installation"): + _device.resolved(torch, device="mps", precision="fp32", connection_name="detector") + + +def _broken(*_: object, **__: object) -> object: + raise ValueError("half an installation") + + def test_mps_on_a_machine_without_metal_falls_back_to_the_cpu( caplog: pytest.LogCaptureFixture, ) -> None: From ed1256d722c7d4a7f0b681ccdebb9bdc57a307d6 Mon Sep 17 00:00:00 2001 From: Armando Anaya Date: Fri, 14 Aug 2026 02:35:35 -0700 Subject: [PATCH 4/4] style: functools.cache, and ruff's formatting of the widened stub cf. #573 --- src/visionset/inference/_device.py | 4 ++-- tests/inference/stubs.py | 4 +--- tests/inference/test_device.py | 7 ++++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/visionset/inference/_device.py b/src/visionset/inference/_device.py index 4958628d..4972e01c 100644 --- a/src/visionset/inference/_device.py +++ b/src/visionset/inference/_device.py @@ -31,7 +31,7 @@ import logging import os -from functools import lru_cache +from functools import cache from typing import Any, Final from visionset.inference import _fp16 @@ -113,7 +113,7 @@ def _present(torch: Any, device: str) -> bool: return True -@lru_cache(maxsize=None) +@cache def _mps_serves(torch: Any) -> bool: """Whether Metal will really take a tensor, asked by handing it one. diff --git a/tests/inference/stubs.py b/tests/inference/stubs.py index 88d352db..08b9e9da 100644 --- a/tests/inference/stubs.py +++ b/tests/inference/stubs.py @@ -167,9 +167,7 @@ class StubTorch: float16 = "float16" - def __init__( - self, *, cuda: bool = False, mps: bool = False, mps_usable: bool = True - ) -> None: + def __init__(self, *, cuda: bool = False, mps: bool = False, mps_usable: bool = True) -> None: self.nn = SimpleNamespace(functional=Functional()) self.cuda = SimpleNamespace(is_available=lambda: cuda) self.backends = SimpleNamespace(mps=SimpleNamespace(is_available=lambda: mps)) diff --git a/tests/inference/test_device.py b/tests/inference/test_device.py index 38f7451e..9f41e099 100644 --- a/tests/inference/test_device.py +++ b/tests/inference/test_device.py @@ -114,9 +114,10 @@ def test_the_probe_is_not_paid_for_on_a_machine_with_no_metal_at_all() -> None: torch = StubTorch(mps=False) torch.zeros = _never # type: ignore[method-assign] - assert _device.resolved( - torch, device="mps", precision="fp32", connection_name="detector" - ) == ("cpu", False) + assert _device.resolved(torch, device="mps", precision="fp32", connection_name="detector") == ( + "cpu", + False, + ) def _never(*_: object, **__: object) -> object: