Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,15 @@ authoring, maintenance, and infrastructure. Full
numbers, methodology, and caveats:
[benchmark/BENCHMARK.md](benchmark/BENCHMARK.md).

The stack also ships a pinned, containerized lending reference environment,
[`benchmark/frappe_lending/`](benchmark/frappe_lending/README.md) — pinned
containers + lockfile, with independent REST, SQL, and exact table-delta
verification of every write. In the model-free engineering matrix (compiled
and direct-API arms, baseline plus cosmetic drift), it delivered **12/12
correct rows with zero silent wrong writes, zero over-halts, and $0 model
cost**. The paid agent arm and full per-cell trial counts are the next stage
of the matrix.

## Status

Early, and honest about it — maturity is uneven across the surface. The
Expand All @@ -286,6 +295,16 @@ start with [`docs/ENTERPRISE_ARCHITECTURE.md`](docs/ENTERPRISE_ARCHITECTURE.md),
which maps screenshot/credential flows, cryptographic guarantees, hosted
boundaries, and unmet controls.

**Machine-checked claims.** OpenAdapt is the only automation vendor whose
public maturity claims are enforced by CI. Every claim above is registered in
[`claims.yaml`](claims.yaml), tiered (supported / validating / roadmap /
research) and mapped to the specific tests and benchmark artifacts that back
it. CI runs `scripts/validate_claims.py`, which **fails the build whenever a
claim's tier outranks its strongest evidence** and regenerates
[`docs/VERIFICATION.md`](docs/VERIFICATION.md) — the claim-by-claim
verification report — from the registry, so the adjectives in this README
cannot quietly rot.

## Privacy (PHI)

For regulated deployments, PHI scrubbing on the persist/log paths is provided by
Expand Down Expand Up @@ -420,5 +439,10 @@ playwright install chromium # optional: else auto-downloads on first launch
pytest -q
```

Contributions are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md). A
ready-made first contribution: pick a module off the mypy type-debt burn-down
list (`[[tool.mypy.overrides]]` in `pyproject.toml`), tighten its annotations,
and remove it from the list.

The demo GIF is generated from real run artifacts by
`scripts/make_demo_gif.py`. MIT license.
44 changes: 40 additions & 4 deletions openadapt_flow/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,11 +346,17 @@ def _build_and_run_replayer(
)


def _finish_replay(run_dir: Path, report) -> int:
"""Render the run report, print the outcome, and map it to an exit code."""
def _finish_replay(run_dir: Path, report, *, synthetic_demo: bool = False) -> int:
"""Render the run report, print the outcome, and map it to an exit code.

``synthetic_demo`` is True only for the bundled-MockMed demo replay with no
operator ``--param`` overrides (see ``render_run_report``); it softens the
first-run plaintext-PHI warning for known-synthetic demo data and nothing
else.
"""
from openadapt_flow.report import render_run_report

report_md = render_run_report(run_dir)
report_md = render_run_report(run_dir, synthetic_demo=synthetic_demo)
outcome = "success" if report.success else "FAILED"
print(f"Replay {outcome}: {report_md}")
if report.screenshots_may_leave_box:
Expand Down Expand Up @@ -724,7 +730,16 @@ def _cmd_replay(args: argparse.Namespace) -> int:
if stop is not None:
stop()

return _finish_replay(run_dir, report)
# Soften the first-run plaintext-PHI warning ONLY when the CLI itself
# served the bundled synthetic MockMed demo (no --url) and no operator
# values flowed in (--param / --params-file / worklists) — every
# identity-like value is then the recorded fake demo data. Real targets
# or operator-supplied values keep the full warning.
return _finish_replay(
run_dir,
report,
synthetic_demo=(stop is not None and not params and not worklists),
)


def _cmd_run(args: argparse.Namespace) -> int:
Expand Down Expand Up @@ -1573,6 +1588,22 @@ def _add_deployment_flags(
)


def _package_version() -> str:
"""The installed ``openadapt-flow`` distribution version.

Falls back to the source tree's ``openadapt_flow.__version__`` when the
package is not installed as a distribution (e.g. run from a checkout).
"""
from importlib.metadata import PackageNotFoundError, version

try:
return version("openadapt-flow")
except PackageNotFoundError:
from openadapt_flow import __version__

return __version__


def build_parser() -> argparse.ArgumentParser:
"""Build the top-level argument parser."""
parser = argparse.ArgumentParser(
Expand All @@ -1583,6 +1614,11 @@ def build_parser() -> argparse.ArgumentParser:
"re-resolution or governed repair when the interface drifts."
),
)
parser.add_argument(
"--version",
action="version",
version=f"%(prog)s {_package_version()}",
)
sub = parser.add_subparsers(dest="command", required=True)

p = sub.add_parser(
Expand Down
20 changes: 17 additions & 3 deletions openadapt_flow/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def _report_has_identity_like_text(report: RunReport) -> bool:
return any((r.intent or "").strip() for r in report.results)


def _warn_if_plaintext_phi(report: RunReport) -> None:
def _warn_if_plaintext_phi(report: RunReport, *, synthetic_demo: bool = False) -> None:
"""Warn (once) when REPORT.md will contain plaintext identity-like text.

Fires only when scrubbing is *not* active (default ``auto`` with the
Expand All @@ -47,7 +47,17 @@ def _warn_if_plaintext_phi(report: RunReport) -> None:
``on`` fails closed upstream before reaching here. Not a behavior change —
the report is still written; this only makes the plaintext write visible.
``warnings`` dedups per call site, so it is effectively one-time per process.

``synthetic_demo=True`` suppresses the warning. The ONLY caller that sets
it is the ``replay`` CLI, and only when the CLI itself served the bundled
MockMed demo app in-process (no ``--url``) AND the operator supplied no
``--param`` overrides — i.e. the identity-like free text is the recorded
synthetic demo data, so alarming a first-run user about PHI would be
noise. Any replay against a real app, or with operator-supplied values,
keeps the full warning.
"""
if synthetic_demo:
return
if _scrub_mode() == "off" or _text_scrubbing_enabled():
return
if not _report_has_identity_like_text(report):
Expand Down Expand Up @@ -101,12 +111,16 @@ def _before_after_table(result: StepResult) -> list[str]:
]


def render_run_report(run_dir: Path | str) -> Path:
def render_run_report(run_dir: Path | str, *, synthetic_demo: bool = False) -> Path:
"""Render ``REPORT.md`` inside ``run_dir`` from its ``report.json``.

Args:
run_dir: Run directory containing ``report.json`` (and the
``steps/`` / ``heals/`` image folders it references).
synthetic_demo: True ONLY when the caller knows every identity-like
free-text field is bundled synthetic demo data (the CLI's
bundled-MockMed replay with no operator ``--param`` overrides);
suppresses the plaintext-PHI warning, nothing else.

Returns:
Path to the written ``REPORT.md``.
Expand All @@ -118,7 +132,7 @@ def render_run_report(run_dir: Path | str) -> Path:
report = RunReport.model_validate_json(
(run / "report.json").read_text(encoding="utf-8")
)
_warn_if_plaintext_phi(report)
_warn_if_plaintext_phi(report, synthetic_demo=synthetic_demo)

ok_count = sum(1 for r in report.results if r.ok)
icon = "✅" if report.success else "❌"
Expand Down
3 changes: 2 additions & 1 deletion tests/test_backend_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,8 @@ def fake_run(backend, **kwargs):

monkeypatch.setattr(m, "_build_and_run_replayer", fake_run)
monkeypatch.setattr(
"openadapt_flow.report.render_run_report", lambda run_dir: "REPORT.md"
"openadapt_flow.report.render_run_report",
lambda run_dir, **_kw: "REPORT.md",
)
monkeypatch.setattr("openadapt_flow.ir.Workflow.load", lambda bundle: object())

Expand Down
26 changes: 24 additions & 2 deletions tests/test_cli_new_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,9 @@ def run(self, workflow, **kwargs):
monkeypatch.setattr(bs, "ensure_chromium_installed", lambda: None)
monkeypatch.setattr(grounder_mod, "build_grounder", lambda fallback=None: None)
monkeypatch.setattr(remote_mod, "appliance_from_env", lambda: None)
monkeypatch.setattr(report_mod, "render_run_report", lambda run_dir: "REPORT.md")
monkeypatch.setattr(
report_mod, "render_run_report", lambda run_dir, **_kw: "REPORT.md"
)
monkeypatch.setattr(runtime_mod, "Replayer", _FakeReplayer)


Expand Down Expand Up @@ -528,7 +530,9 @@ def __init__(self, backend, **kwargs):
monkeypatch.setattr(
durable_mod, "resume", lambda run_dir, replayer, key=None: _FakeReport()
)
monkeypatch.setattr(report_mod, "render_run_report", lambda run_dir: "REPORT.md")
monkeypatch.setattr(
report_mod, "render_run_report", lambda run_dir, **_kw: "REPORT.md"
)

rc = main(
[
Expand All @@ -545,3 +549,21 @@ def __init__(self, backend, **kwargs):
backend = captured["backend"]
assert type(backend).__name__ == "WindowsBackend"
assert backend.server_url == "http://localhost:5001"


# ---------------------------------------------------------------------------
# --version
# ---------------------------------------------------------------------------


def test_version_flag_prints_version_and_exits_zero(capsys) -> None:
"""``openadapt-flow --version`` prints the version and exits 0 (no subcommand)."""
from openadapt_flow.__main__ import _package_version

with pytest.raises(SystemExit) as excinfo:
main(["--version"])
assert excinfo.value.code == 0
out = capsys.readouterr().out.strip()
assert out == f"openadapt-flow {_package_version()}"
prog, _, ver = out.partition(" ")
assert ver and ver[0].isdigit()
24 changes: 24 additions & 0 deletions tests/test_privacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,3 +296,27 @@ def test_report_md_no_warning_when_scrub_off(monkeypatch, tmp_path: Path):
with _w.catch_warnings():
_w.simplefilter("error", PlaintextPHIWarning)
render_run_report(run_dir) # must not raise


def test_report_md_no_warning_for_bundled_synthetic_demo(monkeypatch, tmp_path: Path):
"""``synthetic_demo=True`` (bundled MockMed, no operator values) => silent.

The replay CLI sets this only when it served the bundled synthetic MockMed
demo itself (no ``--url``) and no ``--param``/worklist values flowed in,
so the identity-like text is known-fake demo data.
"""
from openadapt_flow.report import PlaintextPHIWarning

monkeypatch.setattr(privacy, "_build_provider", lambda: None)
privacy.reset_scrubbers()
run_dir = tmp_path / "run"
run_dir.mkdir()
_phi_report().save(run_dir)

import warnings as _w

with _w.catch_warnings():
_w.simplefilter("error", PlaintextPHIWarning)
md = render_run_report(run_dir, synthetic_demo=True).read_text()
# Only the warning is softened — the report content is unchanged.
assert "John Smith" in md