fix: classify invalid function inputs as user errors instead of runtime crashes

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.13.8"
+version = "2.13.9"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/_cli/_evals/_telemetry.py

@@ -20,6 +20,7 @@
 )
 from uipath.platform.common import UiPathConfig
 from uipath.platform.constants import ENV_TENANT_ID
+from uipath.runtime.errors import UiPathBaseRuntimeError
 from uipath.telemetry._track import is_telemetry_enabled, track_event
 
 logger = logging.getLogger(__name__)
@@ -238,15 +239,15 @@ async def _on_eval_run_updated(self, event: EvalRunUpdatedEvent) -> None:
                 )
 
             if event.exception_details:
-                properties["ErrorType"] = type(
-                    event.exception_details.exception
-                ).__name__
-                properties["ErrorMessage"] = str(event.exception_details.exception)[
-                    :500
-                ]
+                exception = event.exception_details.exception
+                properties["ErrorType"] = type(exception).__name__
+                properties["ErrorMessage"] = str(exception)[:500]
                 properties["IsRuntimeException"] = (
                     event.exception_details.runtime_exception
                 )
+                if isinstance(exception, UiPathBaseRuntimeError):
+                    properties["ErrorCode"] = exception.error_info.code
+                    properties["ErrorCategory"] = exception.error_info.category.value
 
             self._enrich_properties(properties)
 

packages/uipath/src/uipath/eval/runtime/runtime.py

@@ -38,6 +38,7 @@
     UiPathRuntimeStorageProtocol,
 )
 from uipath.runtime.errors import (
+    UiPathBaseRuntimeError,
     UiPathErrorCategory,
     UiPathErrorContract,
 )
@@ -96,6 +97,19 @@
 logger = logging.getLogger(__name__)
 
 
+def _is_user_facing_error(exception: BaseException) -> bool:
+    """Whether an exception is a correctly-reported workload failure.
+
+    User-category errors (e.g. invalid input, business logic failures) are
+    failures of the agent under evaluation, not eval infrastructure crashes,
+    so they must not be flagged as runtime exceptions.
+    """
+    return (
+        isinstance(exception, UiPathBaseRuntimeError)
+        and exception.error_info.category == UiPathErrorCategory.USER
+    )
+
+
 def compute_evaluator_scores(
     evaluation_set_results: list[UiPathEvalRunResult],
     evaluators: Iterable[GenericBaseEvaluator[Any, Any, Any]],
@@ -768,7 +782,13 @@ async def _execute_eval(
                 )
 
             except Exception as e:
-                exception_details = EvalItemExceptionDetails(exception=e)
+                root_exception: Exception = (
+                    e.root_exception if isinstance(e, EvaluationRuntimeException) else e
+                )
+                exception_details = EvalItemExceptionDetails(
+                    exception=root_exception,
+                    runtime_exception=not _is_user_facing_error(root_exception),
+                )
 
                 for evaluator in evaluators:
                     evaluation_run_results.evaluation_run_results.append(
@@ -793,13 +813,6 @@ async def _execute_eval(
                 if isinstance(e, EvaluationRuntimeException):
                     eval_run_updated_event.spans = e.spans
                     eval_run_updated_event.logs = e.logs
-                    if eval_run_updated_event.exception_details:
-                        eval_run_updated_event.exception_details.exception = (
-                            e.root_exception
-                        )
-                        eval_run_updated_event.exception_details.runtime_exception = (
-                            True
-                        )
 
                 await self.event_bus.publish(
                     EvaluationEvents.UPDATE_EVAL_RUN,

packages/uipath/src/uipath/functions/runtime.py

@@ -10,6 +10,8 @@
 from types import ModuleType
 from typing import Any, AsyncGenerator, Callable, Type, cast, get_type_hints
 
+from pydantic import ValidationError
+
 from uipath.runtime import (
     UiPathExecuteOptions,
     UiPathRuntimeEvent,
@@ -35,6 +37,24 @@
 logger = logging.getLogger(__name__)
 
 
+def _format_input_validation_error(
+    function_name: str, input_type: Type[Any], error: Exception
+) -> str:
+    """Build a concise, human-readable message for an input conversion failure."""
+    type_name = getattr(input_type, "__name__", str(input_type))
+    if isinstance(error, ValidationError):
+        issues = "; ".join(
+            f"{'.'.join(str(loc) for loc in err['loc']) or type_name}: {err['msg']}"
+            for err in error.errors()
+        )
+    else:
+        issues = str(error)
+    return (
+        f"Input does not match the expected schema for "
+        f"'{function_name}' ({type_name}): {issues}"
+    )
+
+
 class UiPathFunctionsRuntime:
     """Runtime wrapper for a single Python function with full script executor compatibility."""
 
@@ -141,7 +161,18 @@
             or is_pydantic_model(input_type)
             or (inspect.isclass(input_type) and hasattr(input_type, "__annotations__"))
         ):
-            typed_input = convert_to_class(input_data, cast(Type[Any], input_type))
+            try:
+                typed_input = convert_to_class(input_data, cast(Type[Any], input_type))
+            except (ValidationError, TypeError, ValueError) as e:
+                raise UiPathRuntimeError(
+                    # Closest available code; uipath-runtime 0.12.x has no
+                    # dedicated input-validation error code yet.
+                    UiPathErrorCode.INPUT_INVALID_JSON,
+                    "Invalid input",
+                    _format_input_validation_error(self.function_name, input_type, e),
+                    UiPathErrorCategory.USER,
+                    include_traceback=False,
+                ) from e
             result = await func(typed_input) if is_async else func(typed_input)
         else:
             # Dict/untyped parameter

packages/uipath/tests/cli/eval/test_eval_telemetry.py

@@ -337,6 +337,45 @@ async def test_on_eval_run_updated_failure(self, mock_track_event):
         assert "Test error" in properties["ErrorMessage"]
         assert properties["IsRuntimeException"] is True
 
+    @pytest.mark.asyncio
+    @patch("uipath._cli._evals._telemetry.track_event")
+    async def test_on_eval_run_updated_user_error_includes_classification(
+        self, mock_track_event
+    ):
+        """User-category runtime errors emit ErrorCode/ErrorCategory and are not runtime exceptions."""
+        from uipath.runtime.errors import (
+            UiPathErrorCategory,
+            UiPathErrorCode,
+            UiPathRuntimeError,
+        )
+
+        subscriber = EvalTelemetrySubscriber()
+        error = UiPathRuntimeError(
+            UiPathErrorCode.INPUT_INVALID_JSON,
+            "Invalid input",
+            "Input does not match the expected schema",
+            UiPathErrorCategory.USER,
+            include_traceback=False,
+        )
+        exception_details = EvalItemExceptionDetails(
+            exception=error,
+            runtime_exception=False,
+        )
+        event = self._create_eval_run_updated_event(
+            success=False,
+            exception_details=exception_details,
+        )
+
+        await subscriber._on_eval_run_updated(event)
+
+        call_args = mock_track_event.call_args
+        assert call_args[0][0] == EVAL_RUN_FAILED
+        properties = call_args[0][1]
+        assert properties["ErrorType"] == "UiPathRuntimeError"
+        assert properties["ErrorCode"] == "Python.INPUT_INVALID_JSON"
+        assert properties["ErrorCategory"] == "User"
+        assert properties["IsRuntimeException"] is False
+
     @pytest.mark.asyncio
     @patch("uipath._cli._evals._telemetry.track_event")
     async def test_on_eval_run_updated_with_scores(self, mock_track_event):

packages/uipath/tests/cli/eval/test_runtime_error_classification.py

@@ -0,0 +1,171 @@
+"""Tests for classifying eval execution errors as user vs runtime failures."""
+
+import uuid
+from pathlib import Path
+from typing import Any, AsyncGenerator
+
+from uipath.core.events import EventBus
+from uipath.core.tracing import UiPathTraceManager
+from uipath.eval.helpers import EvalHelpers
+from uipath.eval.runtime import UiPathEvalContext, evaluate
+from uipath.eval.runtime.events import EvalRunUpdatedEvent, EvaluationEvents
+from uipath.eval.runtime.runtime import _is_user_facing_error
+from uipath.runtime import (
+    UiPathExecuteOptions,
+    UiPathRuntimeEvent,
+    UiPathRuntimeFactorySettings,
+    UiPathRuntimeProtocol,
+    UiPathRuntimeResult,
+    UiPathRuntimeStorageProtocol,
+    UiPathStreamOptions,
+)
+from uipath.runtime.errors import (
+    UiPathErrorCategory,
+    UiPathErrorCode,
+    UiPathRuntimeError,
+)
+from uipath.runtime.schema import UiPathRuntimeSchema
+
+
+def _make_error(category: UiPathErrorCategory) -> UiPathRuntimeError:
+    return UiPathRuntimeError(
+        UiPathErrorCode.FUNCTION_EXECUTION_ERROR,
+        "Some failure",
+        "details",
+        category,
+        include_traceback=False,
+    )
+
+
+def test_user_category_error_is_user_facing():
+    assert _is_user_facing_error(_make_error(UiPathErrorCategory.USER)) is True
+
+
+def test_system_category_error_is_not_user_facing():
+    assert _is_user_facing_error(_make_error(UiPathErrorCategory.SYSTEM)) is False
+
+
+def test_unknown_category_error_is_not_user_facing():
+    assert _is_user_facing_error(_make_error(UiPathErrorCategory.UNKNOWN)) is False
+
+
+def test_plain_exception_is_not_user_facing():
+    assert _is_user_facing_error(ValueError("boom")) is False
+
+
+class _FailingRuntime:
+    """Runtime whose execution always raises the configured error."""
+
+    def __init__(self, error: Exception):
+        self._error = error
+
+    async def execute(
+        self,
+        input: dict[str, Any] | None = None,
+        options: UiPathExecuteOptions | None = None,
+    ) -> UiPathRuntimeResult:
+        raise self._error
+
+    async def stream(
+        self,
+        input: dict[str, Any] | None = None,
+        options: UiPathStreamOptions | None = None,
+    ) -> AsyncGenerator[UiPathRuntimeEvent, None]:
+        raise self._error
+        yield  # unreachable; makes this an async generator
+
+    async def get_schema(self) -> UiPathRuntimeSchema:
+        return UiPathRuntimeSchema(
+            filePath="test.py",
+            uniqueId="test",
+            type="workflow",
+            input={"type": "object", "properties": {}},
+            output={"type": "object", "properties": {}},
+        )
+
+    async def dispose(self) -> None:
+        pass
+
+
+class _FailingFactory:
+    def __init__(self, error: Exception):
+        self._error = error
+
+    def discover_entrypoints(self) -> list[str]:
+        return ["test"]
+
+    async def get_storage(self) -> UiPathRuntimeStorageProtocol | None:
+        return None
+
+    async def get_settings(self) -> UiPathRuntimeFactorySettings | None:
+        return None
+
+    async def new_runtime(
+        self, entrypoint: str, runtime_id: str, **kwargs
+    ) -> UiPathRuntimeProtocol:
+        return _FailingRuntime(self._error)
+
+    async def dispose(self) -> None:
+        pass
+
+
+async def _run_failing_eval(error: Exception) -> list[EvalRunUpdatedEvent]:
+    """Run an eval whose execution raises, returning captured run-updated events."""
+    event_bus = EventBus()
+    trace_manager = UiPathTraceManager()
+    captured: list[EvalRunUpdatedEvent] = []
+
+    async def capture(event: EvalRunUpdatedEvent) -> None:
+        captured.append(event)
+
+    event_bus.subscribe(EvaluationEvents.UPDATE_EVAL_RUN, capture)
+
+    factory = _FailingFactory(error)
+    eval_set_path = str(Path(__file__).parent / "evals" / "eval-sets" / "default.json")
+    evaluation_set, _ = EvalHelpers.load_eval_set(eval_set_path)
+    runtime = await factory.new_runtime("test", "test-runtime-id")
+    runtime_schema = await runtime.get_schema()
+    evaluators = await EvalHelpers.load_evaluators(
+        eval_set_path, evaluation_set, agent_model=None
+    )
+
+    context = UiPathEvalContext()
+    context.execution_id = str(uuid.uuid4())
+    context.evaluation_set = evaluation_set
+    context.runtime_schema = runtime_schema
+    context.evaluators = evaluators
+
+    await evaluate(factory, trace_manager, context, event_bus)
+    await event_bus.wait_for_all()
+    return captured
+
+
+async def test_user_error_execution_is_not_a_runtime_exception():
+    """A USER-category failure is reported unwrapped with runtime_exception=False."""
+    error = UiPathRuntimeError(
+        UiPathErrorCode.INPUT_INVALID_JSON,
+        "Invalid input",
+        "Input does not match the expected schema",
+        UiPathErrorCategory.USER,
+        include_traceback=False,
+    )
+    events = await _run_failing_eval(error)
+
+    failed = [e for e in events if not e.success]
+    assert failed
+    details = failed[0].exception_details
+    assert details is not None
+    assert details.exception is error
+    assert details.runtime_exception is False
+
+
+async def test_unexpected_error_execution_is_a_runtime_exception():
+    """A non-user failure in the execution path is flagged as a runtime exception."""
+    events = await _run_failing_eval(RuntimeError("infrastructure broke"))
+
+    failed = [e for e in events if not e.success]
+    assert failed
+    details = failed[0].exception_details
+    assert details is not None
+    assert isinstance(details.exception, RuntimeError)
+    assert details.runtime_exception is True