From 4ec33cfd9db64addd0e9f0f6b7af042032b412cc Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Wed, 8 Jul 2026 15:32:02 -0700 Subject: [PATCH 1/4] fix: classify invalid function inputs as user errors instead of runtime crashes Input that fails Pydantic/dataclass conversion in the functions runtime previously surfaced as a generic FUNCTION_EXECUTION_ERROR with a full Python traceback embedded in the error detail, and the eval runtime unconditionally flagged it as a runtime exception. This made test-data problems indistinguishable from eval infrastructure crashes in App Insights and triggered SRE alerting on EvalRun.Failed events (SRE-618832). - functions runtime: catch input conversion failures at convert_to_class and raise a USER-category "Invalid input" error with a concise per-field message and no traceback - eval runtime: only set runtime_exception=True when the root error is not a user-category UiPathBaseRuntimeError - eval telemetry: emit ErrorCode and ErrorCategory properties so alerts can filter user-caused failures without org-ID allowlists Co-Authored-By: Claude Fable 5 --- .../src/uipath/_cli/_evals/_telemetry.py | 13 ++-- .../uipath/src/uipath/eval/runtime/runtime.py | 16 +++- .../uipath/src/uipath/functions/runtime.py | 33 +++++++- .../tests/cli/eval/test_eval_telemetry.py | 39 ++++++++++ .../eval/test_runtime_error_classification.py | 34 ++++++++ .../tests/functions/test_input_validation.py | 77 +++++++++++++++++++ 6 files changed, 204 insertions(+), 8 deletions(-) create mode 100644 packages/uipath/tests/cli/eval/test_runtime_error_classification.py create mode 100644 packages/uipath/tests/functions/test_input_validation.py diff --git a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py index b3dfb160e..64843972b 100644 --- a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py +++ b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py @@ -20,6 +20,7 @@ ) from uipath.platform.common import UiPathConfig from uipath.platform.constants import ENV_TENANT_ID +from uipath.runtime.errors import UiPathBaseRuntimeError from uipath.telemetry._track import is_telemetry_enabled, track_event logger = logging.getLogger(__name__) @@ -238,15 +239,15 @@ async def _on_eval_run_updated(self, event: EvalRunUpdatedEvent) -> None: ) if event.exception_details: - properties["ErrorType"] = type( - event.exception_details.exception - ).__name__ - properties["ErrorMessage"] = str(event.exception_details.exception)[ - :500 - ] + exception = event.exception_details.exception + properties["ErrorType"] = type(exception).__name__ + properties["ErrorMessage"] = str(exception)[:500] properties["IsRuntimeException"] = ( event.exception_details.runtime_exception ) + if isinstance(exception, UiPathBaseRuntimeError): + properties["ErrorCode"] = exception.error_info.code + properties["ErrorCategory"] = exception.error_info.category.value self._enrich_properties(properties) diff --git a/packages/uipath/src/uipath/eval/runtime/runtime.py b/packages/uipath/src/uipath/eval/runtime/runtime.py index 124eedf47..e80912e7b 100644 --- a/packages/uipath/src/uipath/eval/runtime/runtime.py +++ b/packages/uipath/src/uipath/eval/runtime/runtime.py @@ -38,6 +38,7 @@ UiPathRuntimeStorageProtocol, ) from uipath.runtime.errors import ( + UiPathBaseRuntimeError, UiPathErrorCategory, UiPathErrorContract, ) @@ -96,6 +97,19 @@ logger = logging.getLogger(__name__) +def _is_user_facing_error(exception: BaseException) -> bool: + """Whether an exception is a correctly-reported workload failure. + + User-category errors (e.g. invalid input, business logic failures) are + failures of the agent under evaluation, not eval infrastructure crashes, + so they must not be flagged as runtime exceptions. + """ + return ( + isinstance(exception, UiPathBaseRuntimeError) + and exception.error_info.category == UiPathErrorCategory.USER + ) + + def compute_evaluator_scores( evaluation_set_results: list[UiPathEvalRunResult], evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]], @@ -798,7 +812,7 @@ async def _execute_eval( e.root_exception ) eval_run_updated_event.exception_details.runtime_exception = ( - True + not _is_user_facing_error(e.root_exception) ) await self.event_bus.publish( diff --git a/packages/uipath/src/uipath/functions/runtime.py b/packages/uipath/src/uipath/functions/runtime.py index 3467d72b5..2e4b1045b 100644 --- a/packages/uipath/src/uipath/functions/runtime.py +++ b/packages/uipath/src/uipath/functions/runtime.py @@ -10,6 +10,8 @@ from types import ModuleType from typing import Any, AsyncGenerator, Callable, Type, cast, get_type_hints +from pydantic import ValidationError + from uipath.runtime import ( UiPathExecuteOptions, UiPathRuntimeEvent, @@ -35,6 +37,24 @@ logger = logging.getLogger(__name__) +def _format_input_validation_error( + function_name: str, input_type: Type[Any], error: Exception +) -> str: + """Build a concise, human-readable message for an input conversion failure.""" + type_name = getattr(input_type, "__name__", str(input_type)) + if isinstance(error, ValidationError): + issues = "; ".join( + f"{'.'.join(str(loc) for loc in err['loc']) or type_name}: {err['msg']}" + for err in error.errors() + ) + else: + issues = str(error) + return ( + f"Input does not match the expected schema for " + f"'{function_name}' ({type_name}): {issues}" + ) + + class UiPathFunctionsRuntime: """Runtime wrapper for a single Python function with full script executor compatibility.""" @@ -141,7 +161,18 @@ async def _execute_function( or is_pydantic_model(input_type) or (inspect.isclass(input_type) and hasattr(input_type, "__annotations__")) ): - typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + try: + typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + except (ValidationError, TypeError, ValueError) as e: + raise UiPathRuntimeError( + # Closest available code; uipath-runtime 0.12.x has no + # dedicated input-validation error code yet. + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + _format_input_validation_error(self.function_name, input_type, e), + UiPathErrorCategory.USER, + include_traceback=False, + ) from e result = await func(typed_input) if is_async else func(typed_input) else: # Dict/untyped parameter diff --git a/packages/uipath/tests/cli/eval/test_eval_telemetry.py b/packages/uipath/tests/cli/eval/test_eval_telemetry.py index c0e57490c..fdcaf4a85 100644 --- a/packages/uipath/tests/cli/eval/test_eval_telemetry.py +++ b/packages/uipath/tests/cli/eval/test_eval_telemetry.py @@ -337,6 +337,45 @@ async def test_on_eval_run_updated_failure(self, mock_track_event): assert "Test error" in properties["ErrorMessage"] assert properties["IsRuntimeException"] is True + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_on_eval_run_updated_user_error_includes_classification( + self, mock_track_event + ): + """User-category runtime errors emit ErrorCode/ErrorCategory and are not runtime exceptions.""" + from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, + ) + + subscriber = EvalTelemetrySubscriber() + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + exception_details = EvalItemExceptionDetails( + exception=error, + runtime_exception=False, + ) + event = self._create_eval_run_updated_event( + success=False, + exception_details=exception_details, + ) + + await subscriber._on_eval_run_updated(event) + + call_args = mock_track_event.call_args + assert call_args[0][0] == EVAL_RUN_FAILED + properties = call_args[0][1] + assert properties["ErrorType"] == "UiPathRuntimeError" + assert properties["ErrorCode"] == "Python.INPUT_INVALID_JSON" + assert properties["ErrorCategory"] == "User" + assert properties["IsRuntimeException"] is False + @pytest.mark.asyncio @patch("uipath._cli._evals._telemetry.track_event") async def test_on_eval_run_updated_with_scores(self, mock_track_event): diff --git a/packages/uipath/tests/cli/eval/test_runtime_error_classification.py b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py new file mode 100644 index 000000000..2c532f248 --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py @@ -0,0 +1,34 @@ +"""Tests for classifying eval execution errors as user vs runtime failures.""" + +from uipath.eval.runtime.runtime import _is_user_facing_error +from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, +) + + +def _make_error(category: UiPathErrorCategory) -> UiPathRuntimeError: + return UiPathRuntimeError( + UiPathErrorCode.FUNCTION_EXECUTION_ERROR, + "Some failure", + "details", + category, + include_traceback=False, + ) + + +def test_user_category_error_is_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.USER)) is True + + +def test_system_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.SYSTEM)) is False + + +def test_unknown_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.UNKNOWN)) is False + + +def test_plain_exception_is_not_user_facing(): + assert _is_user_facing_error(ValueError("boom")) is False diff --git a/packages/uipath/tests/functions/test_input_validation.py b/packages/uipath/tests/functions/test_input_validation.py new file mode 100644 index 000000000..bb27a6d94 --- /dev/null +++ b/packages/uipath/tests/functions/test_input_validation.py @@ -0,0 +1,77 @@ +"""Tests that invalid inputs surface as classified user errors, not crashes.""" + +import textwrap + +import pytest + +from uipath.functions.runtime import UiPathFunctionsRuntime +from uipath.runtime.errors import UiPathErrorCategory, UiPathRuntimeError + + +@pytest.fixture +def calculator_module(tmp_path): + """Create a module with a Pydantic-typed entrypoint.""" + (tmp_path / "calculator.py").write_text( + textwrap.dedent("""\ + from pydantic import BaseModel + + + class CalculatorInput(BaseModel): + a: int + b: int + + + class CalculatorOutput(BaseModel): + result: int + + + async def main(input: CalculatorInput) -> CalculatorOutput: + return CalculatorOutput(result=input.a + input.b) + """) + ) + return tmp_path / "calculator.py" + + +@pytest.mark.asyncio +async def test_valid_input_executes(calculator_module): + """Sanity check: well-formed input still executes normally.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + result = await runtime.execute({"a": 1, "b": 2}) + assert result.output == {"result": 3} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_input", + [ + {"a": "hello", "b": 2}, + {"a": None, "b": 2}, + {"a": [1, 2], "b": 2}, + {"a": {"x": 1}, "b": 2}, + ], +) +async def test_invalid_input_raises_user_error(calculator_module, bad_input): + """Schema-mismatched input yields a USER-category error without a traceback.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute(bad_input) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.code == "Python.INPUT_INVALID_JSON" + assert error_info.title == "Invalid input" + assert "CalculatorInput" in error_info.detail + assert "main" in error_info.detail + assert "Traceback" not in error_info.detail + + +@pytest.mark.asyncio +async def test_invalid_input_error_lists_offending_fields(calculator_module): + """The error detail names each invalid field with the validation message.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"a": "hello", "b": "world"}) + + detail = exc_info.value.error_info.detail + assert "a:" in detail + assert "b:" in detail From 3690cebd85e8f8a6468b28cc1aeb44cbc7a31bdd Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 9 Jul 2026 14:30:12 -0700 Subject: [PATCH 2/4] chore: bump uipath to 2.13.9 2.13.8 is reserved for #1801, which merges first per its companion-PR sequence. Co-Authored-By: Claude Fable 5 --- packages/uipath/pyproject.toml | 2 +- packages/uipath/uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 2ca26e087..9d5f87757 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.7" +version = "2.13.9" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 19081c3f3..1d0b7726f 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.7" +version = "2.13.9" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, From c0bbd5c899f65027e010e5a0e93b6fd6fd3e2743 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 9 Jul 2026 15:06:24 -0700 Subject: [PATCH 3/4] fix: classify all eval failure-path exceptions, not just wrapped ones Non-EvaluationRuntimeException errors previously fell through with the model default runtime_exception=False, hiding genuine infrastructure failures in telemetry. Unwrap the root exception once and classify every failure uniformly: runtime exception unless user-category. Addresses PR review feedback. Co-Authored-By: Claude Fable 5 --- .../uipath/src/uipath/eval/runtime/runtime.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/uipath/src/uipath/eval/runtime/runtime.py b/packages/uipath/src/uipath/eval/runtime/runtime.py index e80912e7b..f4e3cbbb8 100644 --- a/packages/uipath/src/uipath/eval/runtime/runtime.py +++ b/packages/uipath/src/uipath/eval/runtime/runtime.py @@ -782,7 +782,13 @@ async def _execute_eval( ) except Exception as e: - exception_details = EvalItemExceptionDetails(exception=e) + root_exception: Exception = ( + e.root_exception if isinstance(e, EvaluationRuntimeException) else e + ) + exception_details = EvalItemExceptionDetails( + exception=root_exception, + runtime_exception=not _is_user_facing_error(root_exception), + ) for evaluator in evaluators: evaluation_run_results.evaluation_run_results.append( @@ -807,13 +813,6 @@ async def _execute_eval( if isinstance(e, EvaluationRuntimeException): eval_run_updated_event.spans = e.spans eval_run_updated_event.logs = e.logs - if eval_run_updated_event.exception_details: - eval_run_updated_event.exception_details.exception = ( - e.root_exception - ) - eval_run_updated_event.exception_details.runtime_exception = ( - not _is_user_facing_error(e.root_exception) - ) await self.event_bus.publish( EvaluationEvents.UPDATE_EVAL_RUN, From c6c7d60715ef1dcaab2f5b9ca2426568eba97391 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Thu, 9 Jul 2026 15:26:21 -0700 Subject: [PATCH 4/4] test: cover eval failure-path classification and non-Pydantic input errors SonarCloud new-code coverage was 85.7% (<90% gate): the except-path classification in the eval runtime and the TypeError branch of the input validation message formatter were untested. Add an end-to-end failing-execution eval test asserting runtime_exception classification for user vs unexpected errors, and a dataclass missing-field test. Co-Authored-By: Claude Fable 5 --- .../eval/test_runtime_error_classification.py | 137 ++++++++++++++++++ .../tests/functions/test_input_validation.py | 29 ++++ 2 files changed, 166 insertions(+) diff --git a/packages/uipath/tests/cli/eval/test_runtime_error_classification.py b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py index 2c532f248..721092b4c 100644 --- a/packages/uipath/tests/cli/eval/test_runtime_error_classification.py +++ b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py @@ -1,11 +1,30 @@ """Tests for classifying eval execution errors as user vs runtime failures.""" +import uuid +from pathlib import Path +from typing import Any, AsyncGenerator + +from uipath.core.events import EventBus +from uipath.core.tracing import UiPathTraceManager +from uipath.eval.helpers import EvalHelpers +from uipath.eval.runtime import UiPathEvalContext, evaluate +from uipath.eval.runtime.events import EvalRunUpdatedEvent, EvaluationEvents from uipath.eval.runtime.runtime import _is_user_facing_error +from uipath.runtime import ( + UiPathExecuteOptions, + UiPathRuntimeEvent, + UiPathRuntimeFactorySettings, + UiPathRuntimeProtocol, + UiPathRuntimeResult, + UiPathRuntimeStorageProtocol, + UiPathStreamOptions, +) from uipath.runtime.errors import ( UiPathErrorCategory, UiPathErrorCode, UiPathRuntimeError, ) +from uipath.runtime.schema import UiPathRuntimeSchema def _make_error(category: UiPathErrorCategory) -> UiPathRuntimeError: @@ -32,3 +51,121 @@ def test_unknown_category_error_is_not_user_facing(): def test_plain_exception_is_not_user_facing(): assert _is_user_facing_error(ValueError("boom")) is False + + +class _FailingRuntime: + """Runtime whose execution always raises the configured error.""" + + def __init__(self, error: Exception): + self._error = error + + async def execute( + self, + input: dict[str, Any] | None = None, + options: UiPathExecuteOptions | None = None, + ) -> UiPathRuntimeResult: + raise self._error + + async def stream( + self, + input: dict[str, Any] | None = None, + options: UiPathStreamOptions | None = None, + ) -> AsyncGenerator[UiPathRuntimeEvent, None]: + raise self._error + yield # unreachable; makes this an async generator + + async def get_schema(self) -> UiPathRuntimeSchema: + return UiPathRuntimeSchema( + filePath="test.py", + uniqueId="test", + type="workflow", + input={"type": "object", "properties": {}}, + output={"type": "object", "properties": {}}, + ) + + async def dispose(self) -> None: + pass + + +class _FailingFactory: + def __init__(self, error: Exception): + self._error = error + + def discover_entrypoints(self) -> list[str]: + return ["test"] + + async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: + return None + + async def get_settings(self) -> UiPathRuntimeFactorySettings | None: + return None + + async def new_runtime( + self, entrypoint: str, runtime_id: str, **kwargs + ) -> UiPathRuntimeProtocol: + return _FailingRuntime(self._error) + + async def dispose(self) -> None: + pass + + +async def _run_failing_eval(error: Exception) -> list[EvalRunUpdatedEvent]: + """Run an eval whose execution raises, returning captured run-updated events.""" + event_bus = EventBus() + trace_manager = UiPathTraceManager() + captured: list[EvalRunUpdatedEvent] = [] + + async def capture(event: EvalRunUpdatedEvent) -> None: + captured.append(event) + + event_bus.subscribe(EvaluationEvents.UPDATE_EVAL_RUN, capture) + + factory = _FailingFactory(error) + eval_set_path = str(Path(__file__).parent / "evals" / "eval-sets" / "default.json") + evaluation_set, _ = EvalHelpers.load_eval_set(eval_set_path) + runtime = await factory.new_runtime("test", "test-runtime-id") + runtime_schema = await runtime.get_schema() + evaluators = await EvalHelpers.load_evaluators( + eval_set_path, evaluation_set, agent_model=None + ) + + context = UiPathEvalContext() + context.execution_id = str(uuid.uuid4()) + context.evaluation_set = evaluation_set + context.runtime_schema = runtime_schema + context.evaluators = evaluators + + await evaluate(factory, trace_manager, context, event_bus) + await event_bus.wait_for_all() + return captured + + +async def test_user_error_execution_is_not_a_runtime_exception(): + """A USER-category failure is reported unwrapped with runtime_exception=False.""" + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + events = await _run_failing_eval(error) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert details.exception is error + assert details.runtime_exception is False + + +async def test_unexpected_error_execution_is_a_runtime_exception(): + """A non-user failure in the execution path is flagged as a runtime exception.""" + events = await _run_failing_eval(RuntimeError("infrastructure broke")) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert isinstance(details.exception, RuntimeError) + assert details.runtime_exception is True diff --git a/packages/uipath/tests/functions/test_input_validation.py b/packages/uipath/tests/functions/test_input_validation.py index bb27a6d94..e76ca922e 100644 --- a/packages/uipath/tests/functions/test_input_validation.py +++ b/packages/uipath/tests/functions/test_input_validation.py @@ -65,6 +65,35 @@ async def test_invalid_input_raises_user_error(calculator_module, bad_input): assert "Traceback" not in error_info.detail +@pytest.mark.asyncio +async def test_missing_dataclass_field_raises_user_error(tmp_path): + """Non-Pydantic conversion failures (TypeError) are classified the same way.""" + (tmp_path / "shipping.py").write_text( + textwrap.dedent("""\ + from dataclasses import dataclass + + + @dataclass + class ShippingInput: + address: str + zip_code: str + + + async def main(input: ShippingInput) -> dict: + return {"ok": True} + """) + ) + runtime = UiPathFunctionsRuntime(str(tmp_path / "shipping.py"), "main", "shipping") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"address": "1 Main St"}) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.title == "Invalid input" + assert "ShippingInput" in error_info.detail + assert "zip_code" in error_info.detail + + @pytest.mark.asyncio async def test_invalid_input_error_lists_offending_fields(calculator_module): """The error detail names each invalid field with the validation message."""