diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 1732eb0ea..9d5f87757 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.8" +version = "2.13.9" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py index b3dfb160e..64843972b 100644 --- a/packages/uipath/src/uipath/_cli/_evals/_telemetry.py +++ b/packages/uipath/src/uipath/_cli/_evals/_telemetry.py @@ -20,6 +20,7 @@ ) from uipath.platform.common import UiPathConfig from uipath.platform.constants import ENV_TENANT_ID +from uipath.runtime.errors import UiPathBaseRuntimeError from uipath.telemetry._track import is_telemetry_enabled, track_event logger = logging.getLogger(__name__) @@ -238,15 +239,15 @@ async def _on_eval_run_updated(self, event: EvalRunUpdatedEvent) -> None: ) if event.exception_details: - properties["ErrorType"] = type( - event.exception_details.exception - ).__name__ - properties["ErrorMessage"] = str(event.exception_details.exception)[ - :500 - ] + exception = event.exception_details.exception + properties["ErrorType"] = type(exception).__name__ + properties["ErrorMessage"] = str(exception)[:500] properties["IsRuntimeException"] = ( event.exception_details.runtime_exception ) + if isinstance(exception, UiPathBaseRuntimeError): + properties["ErrorCode"] = exception.error_info.code + properties["ErrorCategory"] = exception.error_info.category.value self._enrich_properties(properties) diff --git a/packages/uipath/src/uipath/eval/runtime/runtime.py b/packages/uipath/src/uipath/eval/runtime/runtime.py index 124eedf47..f4e3cbbb8 100644 --- a/packages/uipath/src/uipath/eval/runtime/runtime.py +++ b/packages/uipath/src/uipath/eval/runtime/runtime.py @@ -38,6 +38,7 @@ UiPathRuntimeStorageProtocol, ) from uipath.runtime.errors import ( + UiPathBaseRuntimeError, UiPathErrorCategory, UiPathErrorContract, ) @@ -96,6 +97,19 @@ logger = logging.getLogger(__name__) +def _is_user_facing_error(exception: BaseException) -> bool: + """Whether an exception is a correctly-reported workload failure. + + User-category errors (e.g. invalid input, business logic failures) are + failures of the agent under evaluation, not eval infrastructure crashes, + so they must not be flagged as runtime exceptions. + """ + return ( + isinstance(exception, UiPathBaseRuntimeError) + and exception.error_info.category == UiPathErrorCategory.USER + ) + + def compute_evaluator_scores( evaluation_set_results: list[UiPathEvalRunResult], evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]], @@ -768,7 +782,13 @@ async def _execute_eval( ) except Exception as e: - exception_details = EvalItemExceptionDetails(exception=e) + root_exception: Exception = ( + e.root_exception if isinstance(e, EvaluationRuntimeException) else e + ) + exception_details = EvalItemExceptionDetails( + exception=root_exception, + runtime_exception=not _is_user_facing_error(root_exception), + ) for evaluator in evaluators: evaluation_run_results.evaluation_run_results.append( @@ -793,13 +813,6 @@ async def _execute_eval( if isinstance(e, EvaluationRuntimeException): eval_run_updated_event.spans = e.spans eval_run_updated_event.logs = e.logs - if eval_run_updated_event.exception_details: - eval_run_updated_event.exception_details.exception = ( - e.root_exception - ) - eval_run_updated_event.exception_details.runtime_exception = ( - True - ) await self.event_bus.publish( EvaluationEvents.UPDATE_EVAL_RUN, diff --git a/packages/uipath/src/uipath/functions/runtime.py b/packages/uipath/src/uipath/functions/runtime.py index 3467d72b5..2e4b1045b 100644 --- a/packages/uipath/src/uipath/functions/runtime.py +++ b/packages/uipath/src/uipath/functions/runtime.py @@ -10,6 +10,8 @@ from types import ModuleType from typing import Any, AsyncGenerator, Callable, Type, cast, get_type_hints +from pydantic import ValidationError + from uipath.runtime import ( UiPathExecuteOptions, UiPathRuntimeEvent, @@ -35,6 +37,24 @@ logger = logging.getLogger(__name__) +def _format_input_validation_error( + function_name: str, input_type: Type[Any], error: Exception +) -> str: + """Build a concise, human-readable message for an input conversion failure.""" + type_name = getattr(input_type, "__name__", str(input_type)) + if isinstance(error, ValidationError): + issues = "; ".join( + f"{'.'.join(str(loc) for loc in err['loc']) or type_name}: {err['msg']}" + for err in error.errors() + ) + else: + issues = str(error) + return ( + f"Input does not match the expected schema for " + f"'{function_name}' ({type_name}): {issues}" + ) + + class UiPathFunctionsRuntime: """Runtime wrapper for a single Python function with full script executor compatibility.""" @@ -141,7 +161,18 @@ async def _execute_function( or is_pydantic_model(input_type) or (inspect.isclass(input_type) and hasattr(input_type, "__annotations__")) ): - typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + try: + typed_input = convert_to_class(input_data, cast(Type[Any], input_type)) + except (ValidationError, TypeError, ValueError) as e: + raise UiPathRuntimeError( + # Closest available code; uipath-runtime 0.12.x has no + # dedicated input-validation error code yet. + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + _format_input_validation_error(self.function_name, input_type, e), + UiPathErrorCategory.USER, + include_traceback=False, + ) from e result = await func(typed_input) if is_async else func(typed_input) else: # Dict/untyped parameter diff --git a/packages/uipath/tests/cli/eval/test_eval_telemetry.py b/packages/uipath/tests/cli/eval/test_eval_telemetry.py index c0e57490c..fdcaf4a85 100644 --- a/packages/uipath/tests/cli/eval/test_eval_telemetry.py +++ b/packages/uipath/tests/cli/eval/test_eval_telemetry.py @@ -337,6 +337,45 @@ async def test_on_eval_run_updated_failure(self, mock_track_event): assert "Test error" in properties["ErrorMessage"] assert properties["IsRuntimeException"] is True + @pytest.mark.asyncio + @patch("uipath._cli._evals._telemetry.track_event") + async def test_on_eval_run_updated_user_error_includes_classification( + self, mock_track_event + ): + """User-category runtime errors emit ErrorCode/ErrorCategory and are not runtime exceptions.""" + from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, + ) + + subscriber = EvalTelemetrySubscriber() + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + exception_details = EvalItemExceptionDetails( + exception=error, + runtime_exception=False, + ) + event = self._create_eval_run_updated_event( + success=False, + exception_details=exception_details, + ) + + await subscriber._on_eval_run_updated(event) + + call_args = mock_track_event.call_args + assert call_args[0][0] == EVAL_RUN_FAILED + properties = call_args[0][1] + assert properties["ErrorType"] == "UiPathRuntimeError" + assert properties["ErrorCode"] == "Python.INPUT_INVALID_JSON" + assert properties["ErrorCategory"] == "User" + assert properties["IsRuntimeException"] is False + @pytest.mark.asyncio @patch("uipath._cli._evals._telemetry.track_event") async def test_on_eval_run_updated_with_scores(self, mock_track_event): diff --git a/packages/uipath/tests/cli/eval/test_runtime_error_classification.py b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py new file mode 100644 index 000000000..721092b4c --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_runtime_error_classification.py @@ -0,0 +1,171 @@ +"""Tests for classifying eval execution errors as user vs runtime failures.""" + +import uuid +from pathlib import Path +from typing import Any, AsyncGenerator + +from uipath.core.events import EventBus +from uipath.core.tracing import UiPathTraceManager +from uipath.eval.helpers import EvalHelpers +from uipath.eval.runtime import UiPathEvalContext, evaluate +from uipath.eval.runtime.events import EvalRunUpdatedEvent, EvaluationEvents +from uipath.eval.runtime.runtime import _is_user_facing_error +from uipath.runtime import ( + UiPathExecuteOptions, + UiPathRuntimeEvent, + UiPathRuntimeFactorySettings, + UiPathRuntimeProtocol, + UiPathRuntimeResult, + UiPathRuntimeStorageProtocol, + UiPathStreamOptions, +) +from uipath.runtime.errors import ( + UiPathErrorCategory, + UiPathErrorCode, + UiPathRuntimeError, +) +from uipath.runtime.schema import UiPathRuntimeSchema + + +def _make_error(category: UiPathErrorCategory) -> UiPathRuntimeError: + return UiPathRuntimeError( + UiPathErrorCode.FUNCTION_EXECUTION_ERROR, + "Some failure", + "details", + category, + include_traceback=False, + ) + + +def test_user_category_error_is_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.USER)) is True + + +def test_system_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.SYSTEM)) is False + + +def test_unknown_category_error_is_not_user_facing(): + assert _is_user_facing_error(_make_error(UiPathErrorCategory.UNKNOWN)) is False + + +def test_plain_exception_is_not_user_facing(): + assert _is_user_facing_error(ValueError("boom")) is False + + +class _FailingRuntime: + """Runtime whose execution always raises the configured error.""" + + def __init__(self, error: Exception): + self._error = error + + async def execute( + self, + input: dict[str, Any] | None = None, + options: UiPathExecuteOptions | None = None, + ) -> UiPathRuntimeResult: + raise self._error + + async def stream( + self, + input: dict[str, Any] | None = None, + options: UiPathStreamOptions | None = None, + ) -> AsyncGenerator[UiPathRuntimeEvent, None]: + raise self._error + yield # unreachable; makes this an async generator + + async def get_schema(self) -> UiPathRuntimeSchema: + return UiPathRuntimeSchema( + filePath="test.py", + uniqueId="test", + type="workflow", + input={"type": "object", "properties": {}}, + output={"type": "object", "properties": {}}, + ) + + async def dispose(self) -> None: + pass + + +class _FailingFactory: + def __init__(self, error: Exception): + self._error = error + + def discover_entrypoints(self) -> list[str]: + return ["test"] + + async def get_storage(self) -> UiPathRuntimeStorageProtocol | None: + return None + + async def get_settings(self) -> UiPathRuntimeFactorySettings | None: + return None + + async def new_runtime( + self, entrypoint: str, runtime_id: str, **kwargs + ) -> UiPathRuntimeProtocol: + return _FailingRuntime(self._error) + + async def dispose(self) -> None: + pass + + +async def _run_failing_eval(error: Exception) -> list[EvalRunUpdatedEvent]: + """Run an eval whose execution raises, returning captured run-updated events.""" + event_bus = EventBus() + trace_manager = UiPathTraceManager() + captured: list[EvalRunUpdatedEvent] = [] + + async def capture(event: EvalRunUpdatedEvent) -> None: + captured.append(event) + + event_bus.subscribe(EvaluationEvents.UPDATE_EVAL_RUN, capture) + + factory = _FailingFactory(error) + eval_set_path = str(Path(__file__).parent / "evals" / "eval-sets" / "default.json") + evaluation_set, _ = EvalHelpers.load_eval_set(eval_set_path) + runtime = await factory.new_runtime("test", "test-runtime-id") + runtime_schema = await runtime.get_schema() + evaluators = await EvalHelpers.load_evaluators( + eval_set_path, evaluation_set, agent_model=None + ) + + context = UiPathEvalContext() + context.execution_id = str(uuid.uuid4()) + context.evaluation_set = evaluation_set + context.runtime_schema = runtime_schema + context.evaluators = evaluators + + await evaluate(factory, trace_manager, context, event_bus) + await event_bus.wait_for_all() + return captured + + +async def test_user_error_execution_is_not_a_runtime_exception(): + """A USER-category failure is reported unwrapped with runtime_exception=False.""" + error = UiPathRuntimeError( + UiPathErrorCode.INPUT_INVALID_JSON, + "Invalid input", + "Input does not match the expected schema", + UiPathErrorCategory.USER, + include_traceback=False, + ) + events = await _run_failing_eval(error) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert details.exception is error + assert details.runtime_exception is False + + +async def test_unexpected_error_execution_is_a_runtime_exception(): + """A non-user failure in the execution path is flagged as a runtime exception.""" + events = await _run_failing_eval(RuntimeError("infrastructure broke")) + + failed = [e for e in events if not e.success] + assert failed + details = failed[0].exception_details + assert details is not None + assert isinstance(details.exception, RuntimeError) + assert details.runtime_exception is True diff --git a/packages/uipath/tests/functions/test_input_validation.py b/packages/uipath/tests/functions/test_input_validation.py new file mode 100644 index 000000000..e76ca922e --- /dev/null +++ b/packages/uipath/tests/functions/test_input_validation.py @@ -0,0 +1,106 @@ +"""Tests that invalid inputs surface as classified user errors, not crashes.""" + +import textwrap + +import pytest + +from uipath.functions.runtime import UiPathFunctionsRuntime +from uipath.runtime.errors import UiPathErrorCategory, UiPathRuntimeError + + +@pytest.fixture +def calculator_module(tmp_path): + """Create a module with a Pydantic-typed entrypoint.""" + (tmp_path / "calculator.py").write_text( + textwrap.dedent("""\ + from pydantic import BaseModel + + + class CalculatorInput(BaseModel): + a: int + b: int + + + class CalculatorOutput(BaseModel): + result: int + + + async def main(input: CalculatorInput) -> CalculatorOutput: + return CalculatorOutput(result=input.a + input.b) + """) + ) + return tmp_path / "calculator.py" + + +@pytest.mark.asyncio +async def test_valid_input_executes(calculator_module): + """Sanity check: well-formed input still executes normally.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + result = await runtime.execute({"a": 1, "b": 2}) + assert result.output == {"result": 3} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_input", + [ + {"a": "hello", "b": 2}, + {"a": None, "b": 2}, + {"a": [1, 2], "b": 2}, + {"a": {"x": 1}, "b": 2}, + ], +) +async def test_invalid_input_raises_user_error(calculator_module, bad_input): + """Schema-mismatched input yields a USER-category error without a traceback.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute(bad_input) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.code == "Python.INPUT_INVALID_JSON" + assert error_info.title == "Invalid input" + assert "CalculatorInput" in error_info.detail + assert "main" in error_info.detail + assert "Traceback" not in error_info.detail + + +@pytest.mark.asyncio +async def test_missing_dataclass_field_raises_user_error(tmp_path): + """Non-Pydantic conversion failures (TypeError) are classified the same way.""" + (tmp_path / "shipping.py").write_text( + textwrap.dedent("""\ + from dataclasses import dataclass + + + @dataclass + class ShippingInput: + address: str + zip_code: str + + + async def main(input: ShippingInput) -> dict: + return {"ok": True} + """) + ) + runtime = UiPathFunctionsRuntime(str(tmp_path / "shipping.py"), "main", "shipping") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"address": "1 Main St"}) + + error_info = exc_info.value.error_info + assert error_info.category == UiPathErrorCategory.USER + assert error_info.title == "Invalid input" + assert "ShippingInput" in error_info.detail + assert "zip_code" in error_info.detail + + +@pytest.mark.asyncio +async def test_invalid_input_error_lists_offending_fields(calculator_module): + """The error detail names each invalid field with the validation message.""" + runtime = UiPathFunctionsRuntime(str(calculator_module), "main", "calculator") + with pytest.raises(UiPathRuntimeError) as exc_info: + await runtime.execute({"a": "hello", "b": "world"}) + + detail = exc_info.value.error_info.detail + assert "a:" in detail + assert "b:" in detail diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 800e2b56a..b14ed59dd 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.8" +version = "2.13.9" source = { editable = "." } dependencies = [ { name = "applicationinsights" },