diff --git a/docs/HYBIM-730-domain-entity-rename.md b/docs/HYBIM-730-domain-entity-rename.md new file mode 100644 index 00000000..5afcf215 --- /dev/null +++ b/docs/HYBIM-730-domain-entity-rename.md @@ -0,0 +1,237 @@ +# Domain Entity Rename: Log Streams → Agent Streams, Metrics → Evaluators + +**Ticket:** [HYBIM-730](https://splunk.atlassian.net/browse/HYBIM-730) +**Branch:** `feat/HYBIM-730-domain-rename` +**Status:** Implemented with full backward compatibility + +--- + +## Summary + +Two core SDK domain entities are being renamed to better reflect their purpose: + +| Old name | New name | Scope | +|----------|----------|-------| +| Log Stream / `LogStream` | Agent Stream / `AgentStream` | Top-level class, service class, convenience functions | +| Metric / `Metric` | Evaluator / `Evaluator` | Base class and all concrete subclasses | + +The old names remain functional but emit `DeprecationWarning` at import time. +They will be removed in a future major release. + +> **Note:** The underlying API endpoints continue to use the previous paths +> (`/log_streams`, `/scorers`) — server-side renaming is tracked separately. + +--- + +## New Public API + +### Agent Streams + +```python +# Primary class (replaces LogStream) +from splunk_ao import AgentStream + +# Create and persist a new agent stream +stream = AgentStream(name="prod-traces", project_name="my-project").create() + +# Retrieve an existing stream +stream = AgentStream.get(name="prod-traces", project_name="my-project") + +# List streams for a project +streams = AgentStream.list(project_name="my-project") + +# Enable evaluators on the stream +from splunk_ao import SplunkAOMetrics +stream.set_metrics([SplunkAOMetrics.correctness, SplunkAOMetrics.completeness]) +``` + +```python +# Service class (replaces LogStreams in splunk_ao.log_streams) +from splunk_ao.agent_streams import AgentStreams + +svc = AgentStreams() +stream = svc.get(name="prod-traces", project_name="my-project") +streams = svc.list(project_name="my-project") +``` + +```python +# Convenience functions (replaces functions in splunk_ao.log_streams) +from splunk_ao.agent_streams import ( + get_agent_stream, + list_agent_streams, + create_agent_stream, + enable_evaluators, +) + +stream = get_agent_stream(name="prod-traces", project_name="my-project") +streams = list_agent_streams(project_name="my-project") +stream = create_agent_stream(name="new-stream", project_name="my-project") + +# Enable evaluators (formerly enable_metrics) +local_evals = enable_evaluators( + agent_stream_name="prod-traces", + project_name="my-project", + metrics=[SplunkAOMetrics.correctness, "completeness"], +) +``` + +### Evaluators + +```python +# Base class (replaces Metric) +from splunk_ao import Evaluator, LlmEvaluator, CodeEvaluator, LocalEvaluator, SplunkAOEvaluator + +# Get an existing evaluator by name or ID +ev = Evaluator.get(name="factuality-checker") + +# List all evaluators +evaluators = Evaluator.list() + +# Access built-in evaluators +ev = Evaluator.evaluators.correctness +ev = Evaluator.evaluators.completeness + +# Create a custom LLM evaluator +llm_ev = LlmEvaluator( + name="response_quality", + prompt="Rate the quality 1-10: {input} -> {output}", + model="gpt-4o-mini", + judges=3, +).create() + +# Create a code-based evaluator +code_ev = CodeEvaluator( + name="custom_scorer", + code="def scorer_fn(step): return 1.0", +).create() + +# Create a local function-based evaluator +def my_fn(trace): + return min(len(getattr(trace, "output", "") or "") / 200.0, 1.0) + +local_ev = LocalEvaluator(name="response_length", scorer_fn=my_fn) +``` + +```python +# Evaluators service class (replaces Metrics in splunk_ao.metrics) +from splunk_ao.evaluators import ( + Evaluators, + create_custom_llm_evaluator, + get_evaluators, + delete_evaluator, +) + +create_custom_llm_evaluator(name="my-eval", user_prompt="Rate this...") +delete_evaluator(name="old-eval") +``` + +### Project methods + +```python +from splunk_ao import Project + +project = Project.get(name="My AI Project") + +# New methods +stream = project.create_agent_stream(name="Production Traces") +streams = project.list_agent_streams() +for stream in project.agent_streams: + print(stream.name) +``` + +--- + +## Backward Compatibility + +All old names continue to work but emit `DeprecationWarning`: + +```python +import warnings +warnings.simplefilter("always", DeprecationWarning) + +# These still work — but warn: +from splunk_ao import LogStream # warns: use AgentStream +from splunk_ao import Metric # warns: use Evaluator +from splunk_ao import LlmMetric # warns: use LlmEvaluator +from splunk_ao import CodeMetric # warns: use CodeEvaluator +from splunk_ao import LocalMetric # warns: use LocalEvaluator +from splunk_ao import SplunkAOMetric # warns: use SplunkAOEvaluator + +# Project deprecated methods — warn: +project.create_log_stream("foo") # warns: use create_agent_stream +project.list_log_streams() # warns: use list_agent_streams +project.logstreams # warns: use agent_streams +``` + +### Migration quick-reference + +| Old import | New import | +|------------|------------| +| `from splunk_ao import LogStream` | `from splunk_ao import AgentStream` | +| `from splunk_ao import Metric` | `from splunk_ao import Evaluator` | +| `from splunk_ao import LlmMetric` | `from splunk_ao import LlmEvaluator` | +| `from splunk_ao import CodeMetric` | `from splunk_ao import CodeEvaluator` | +| `from splunk_ao import LocalMetric` | `from splunk_ao import LocalEvaluator` | +| `from splunk_ao import SplunkAOMetric` | `from splunk_ao import SplunkAOEvaluator` | +| `from splunk_ao.log_stream import LogStream` | `from splunk_ao.agent_stream import AgentStream` | +| `from splunk_ao.log_streams import LogStreams` | `from splunk_ao.agent_streams import AgentStreams` | +| `from splunk_ao.log_streams import get_log_stream` | `from splunk_ao.agent_streams import get_agent_stream` | +| `from splunk_ao.log_streams import list_log_streams` | `from splunk_ao.agent_streams import list_agent_streams` | +| `from splunk_ao.log_streams import create_log_stream` | `from splunk_ao.agent_streams import create_agent_stream` | +| `from splunk_ao.log_streams import enable_metrics` | `from splunk_ao.agent_streams import enable_evaluators` | +| `from splunk_ao.metric import Metric` | `from splunk_ao.evaluator import Evaluator` | +| `from splunk_ao.metrics import Metrics` | `from splunk_ao.evaluators import Evaluators` | +| `project.create_log_stream(name)` | `project.create_agent_stream(name)` | +| `project.list_log_streams()` | `project.list_agent_streams()` | +| `project.logstreams` | `project.agent_streams` | + +--- + +## Files Changed + +| File | Change | +|------|--------| +| `src/splunk_ao/agent_stream.py` | **New** — `AgentStream` subclass of `LogStream` | +| `src/splunk_ao/agent_streams.py` | **New** — `AgentStreams` service class + convenience functions | +| `src/splunk_ao/evaluator.py` | **New** — `Evaluator`, `LlmEvaluator`, `CodeEvaluator`, `LocalEvaluator`, `SplunkAOEvaluator` | +| `src/splunk_ao/evaluators.py` | **New** — `Evaluators` service class + convenience functions | +| `src/splunk_ao/__future__/agent_stream.py` | **New** — deprecation shim | +| `src/splunk_ao/__future__/evaluator.py` | **New** — deprecation shim | +| `src/splunk_ao/__init__.py` | **Modified** — export new names; PEP 562 `__getattr__` for deprecated old names | +| `src/splunk_ao/project.py` | **Modified** — add `create_agent_stream`, `list_agent_streams`, `agent_streams`; deprecate old methods | +| `docs/HYBIM-730-domain-entity-rename.md` | **New** — this document | + +--- + +## Design Decisions + +### Subclassing vs type aliases + +`AgentStream` is implemented as a subclass of `LogStream` (rather than a plain +`= LogStream` alias) so that: +- `AgentStream.__name__` is `"AgentStream"` +- Instances created via `AgentStream(...)` have the new name in repr +- Future implementation divergence is possible without API breakage + +The same pattern applies to `Evaluator` / `LlmEvaluator` / etc. + +### PEP 562 `__getattr__` in `__init__.py` + +Old names (`LogStream`, `Metric`, …) are **not** listed in `__all__`, so they +are invisible to tab-completion and static analysis. They are only reachable +via `__getattr__`, which fires a `DeprecationWarning`. This gives existing code +a graceful migration path without polluting the new API surface. + +### Server-side API paths unchanged + +The API endpoints remain on the old paths (`/log_streams`, `/scorers`). +`AgentStream` / `Evaluator` call into the same service layers (`LogStreams`, +`Metrics`) as before. A separate ticket tracks the server-side rename. + +--- + +## Related + +- [HYBIM-730](https://splunk.atlassian.net/browse/HYBIM-730) — this ticket +- [HYBIM-832](https://splunk.atlassian.net/browse/HYBIM-832) — Galileo → Splunk AO env-var rename +- `docs/OTEL_GALILEO_TO_SPLUNK_AO_RENAME.md` — OTel layer rename diff --git a/src/splunk_ao/__future__/agent_stream.py b/src/splunk_ao/__future__/agent_stream.py new file mode 100644 index 00000000..27a7e092 --- /dev/null +++ b/src/splunk_ao/__future__/agent_stream.py @@ -0,0 +1,14 @@ +"""Deprecated: use splunk_ao.agent_stream instead of splunk_ao.__future__.agent_stream.""" + +import warnings + +warnings.warn( + "Importing from splunk_ao.__future__.agent_stream is deprecated. " + "Use splunk_ao.agent_stream instead.", + DeprecationWarning, + stacklevel=2, +) + +from splunk_ao.agent_stream import AgentStream # noqa: E402 + +__all__ = ["AgentStream"] diff --git a/src/splunk_ao/__future__/evaluator.py b/src/splunk_ao/__future__/evaluator.py new file mode 100644 index 00000000..bb7c5507 --- /dev/null +++ b/src/splunk_ao/__future__/evaluator.py @@ -0,0 +1,20 @@ +"""Deprecated: use splunk_ao.evaluator instead of splunk_ao.__future__.evaluator.""" + +import warnings + +warnings.warn( + "Importing from splunk_ao.__future__.evaluator is deprecated. " + "Use splunk_ao.evaluator instead.", + DeprecationWarning, + stacklevel=2, +) + +from splunk_ao.evaluator import ( # noqa: E402 + CodeEvaluator, + Evaluator, + LlmEvaluator, + LocalEvaluator, + SplunkAOEvaluator, +) + +__all__ = ["CodeEvaluator", "Evaluator", "LlmEvaluator", "LocalEvaluator", "SplunkAOEvaluator"] diff --git a/src/splunk_ao/__init__.py b/src/splunk_ao/__init__.py index 38b0611c..f0047cff 100644 --- a/src/splunk_ao/__init__.py +++ b/src/splunk_ao/__init__.py @@ -16,10 +16,18 @@ from galileo_core.schemas.logging.step import StepType from galileo_core.schemas.logging.trace import Trace from splunk_ao.agent_control import AgentControlTarget, AgentControlTargetUnresolvedError, get_agent_control_target +from splunk_ao.agent_stream import AgentStream from splunk_ao.collaborator import Collaborator, CollaboratorRole from splunk_ao.configuration import Configuration from splunk_ao.dataset import Dataset from splunk_ao.decorator import SplunkAODecorator, log, splunk_ao_context, start_session +from splunk_ao.evaluator import ( + CodeEvaluator, + Evaluator, + LlmEvaluator, + LocalEvaluator, + SplunkAOEvaluator, +) from splunk_ao.exceptions import ( AuthenticationError, BadRequestError, @@ -34,10 +42,8 @@ from splunk_ao.experiment import Experiment from splunk_ao.handlers.agent_control import SplunkAOAgentControlBridge, setup_agent_control_bridge from splunk_ao.integration import Integration -from splunk_ao.log_stream import LogStream from splunk_ao.logger import SplunkAOLogger from splunk_ao.logger.control import ControlAppliesTo, ControlCheckStage, ControlResult, ControlSpan -from splunk_ao.metric import CodeMetric, LlmMetric, LocalMetric, Metric, SplunkAOMetric from splunk_ao.model import Model from splunk_ao.project import Project from splunk_ao.prompt import Prompt @@ -64,6 +70,14 @@ __version__ = "0.1.0" __all__ = [ + # New canonical names (HYBIM-730) + "AgentStream", + "CodeEvaluator", + "Evaluator", + "LlmEvaluator", + "LocalEvaluator", + "SplunkAOEvaluator", + # Stable / unchanged "APIError", "AgentControlTarget", "AgentControlTargetUnresolvedError", @@ -74,7 +88,6 @@ "AzureProvider", "BadRequestError", "BedrockProvider", - "CodeMetric", "Collaborator", "CollaboratorRole", "Configuration", @@ -89,13 +102,9 @@ "Experiment", "ForbiddenError", "Integration", - "LlmMetric", "LlmSpan", - "LocalMetric", - "LogStream", "Message", "MessageRole", - "Metric", "MetricSpec", "MissingConfigurationError", "Model", @@ -118,7 +127,6 @@ "SplunkAOFutureError", "SplunkAOLogger", "SplunkAOLoggerException", - "SplunkAOMetric", "SplunkAOMetrics", "StepType", "StepWithChildSpans", @@ -141,3 +149,42 @@ "splunk_ao_context", "start_session", ] + +# --------------------------------------------------------------------------- +# PEP 562 — deprecated names emit DeprecationWarning on attribute access. +# Old names are NOT in __all__ so they won't appear in tab-completion, but +# they still work for existing code via __getattr__. +# --------------------------------------------------------------------------- +_DEPRECATED_NAMES: dict[str, tuple[str, str]] = { + "LogStream": ("AgentStream", "splunk_ao.agent_stream"), + "Metric": ("Evaluator", "splunk_ao.evaluator"), + "LlmMetric": ("LlmEvaluator", "splunk_ao.evaluator"), + "CodeMetric": ("CodeEvaluator", "splunk_ao.evaluator"), + "LocalMetric": ("LocalEvaluator", "splunk_ao.evaluator"), + "SplunkAOMetric": ("SplunkAOEvaluator", "splunk_ao.evaluator"), +} + +_DEPRECATED_OBJECTS: dict[str, object] = { + "LogStream": AgentStream, + "Metric": Evaluator, + "LlmMetric": LlmEvaluator, + "CodeMetric": CodeEvaluator, + "LocalMetric": LocalEvaluator, + "SplunkAOMetric": SplunkAOEvaluator, +} + + +def __getattr__(name: str) -> object: + if name in _DEPRECATED_NAMES: + import warnings + + new_name, new_module = _DEPRECATED_NAMES[name] + warnings.warn( + f"'splunk_ao.{name}' is deprecated and will be removed in a future release. " + f"Use '{new_name}' from '{new_module}' " + f"(or 'from splunk_ao import {new_name}') instead.", + DeprecationWarning, + stacklevel=2, + ) + return _DEPRECATED_OBJECTS[name] + raise AttributeError(f"module 'splunk_ao' has no attribute {name!r}") diff --git a/src/splunk_ao/agent_stream.py b/src/splunk_ao/agent_stream.py new file mode 100644 index 00000000..8a9f875b --- /dev/null +++ b/src/splunk_ao/agent_stream.py @@ -0,0 +1,61 @@ +""" +Agent Streams — the renamed successor to Log Streams (HYBIM-730). + +``AgentStream`` is the canonical name going forward. ``LogStream`` is kept +as a deprecated alias in ``splunk_ao.__init__`` and will be removed in a +future major release. + +The underlying implementation is shared with ``splunk_ao.log_stream``; the +API endpoints still use the ``/log_streams`` path (server-side rename is +tracked separately). +""" +from __future__ import annotations + +import warnings + +from splunk_ao.log_stream import LogStream + +__all__ = ["AgentStream"] + + +class AgentStream(LogStream): + """ + Object-centric interface for Splunk AO agent streams. + + ``AgentStream`` is the new name for what was previously called a + *Log Stream*. All functionality is identical; only the name has + changed. Use ``AgentStream`` for all new code. + + See ``splunk_ao.log_stream.LogStream`` for the full API reference — + every method, property, and class-method is inherited unchanged. + + Examples + -------- + from splunk_ao import AgentStream + + # Create and persist a new agent stream + stream = AgentStream(name="prod-traces", project_name="my-project").create() + + # Retrieve an existing stream + stream = AgentStream.get(name="prod-traces", project_name="my-project") + + # List streams for a project + streams = AgentStream.list(project_name="my-project") + + # Enable evaluators on the stream + from splunk_ao import SplunkAOMetrics + stream.set_metrics([SplunkAOMetrics.correctness, SplunkAOMetrics.completeness]) + """ + + +# Convenience: expose the deprecated ``LogStream`` name with a warning +def __getattr__(name: str): + if name == "LogStream": + warnings.warn( + "splunk_ao.agent_stream.LogStream is deprecated; " + "import AgentStream from splunk_ao.agent_stream instead.", + DeprecationWarning, + stacklevel=2, + ) + return LogStream + raise AttributeError(f"module 'splunk_ao.agent_stream' has no attribute {name!r}") diff --git a/src/splunk_ao/agent_streams.py b/src/splunk_ao/agent_streams.py new file mode 100644 index 00000000..a104e43b --- /dev/null +++ b/src/splunk_ao/agent_streams.py @@ -0,0 +1,208 @@ +""" +Agent Streams service layer — the renamed successor to Log Streams (HYBIM-730). + +Provides ``AgentStreams`` (service class) and the module-level convenience +functions ``get_agent_stream``, ``list_agent_streams``, ``create_agent_stream``, +and ``enable_evaluators``. + +The old ``log_streams`` module and its symbols remain available but are +deprecated. +""" +from __future__ import annotations + +import builtins +import warnings + +from splunk_ao.agent_stream import AgentStream +from splunk_ao.log_streams import LogStreams, create_log_stream, get_log_stream, list_log_streams +from splunk_ao.resources.types import Unset +from splunk_ao.schema.metrics import LocalMetricConfig, Metric, SplunkAOMetrics + +__all__ = [ + "AgentStreams", + "create_agent_stream", + "enable_evaluators", + "get_agent_stream", + "list_agent_streams", +] + + +class AgentStreams(LogStreams): + """ + Low-level service class for managing agent streams. + + ``AgentStreams`` is the new name for the ``LogStreams`` service class. + Inherits all methods from ``LogStreams`` and returns ``AgentStream`` + instances from listing/retrieval methods. + + Examples + -------- + from splunk_ao.agent_streams import AgentStreams + + svc = AgentStreams() + stream = svc.get(name="prod-traces", project_name="my-project") + streams = svc.list(project_name="my-project") + """ + + +# --------------------------------------------------------------------------- +# Module-level convenience functions +# --------------------------------------------------------------------------- + + +def get_agent_stream( + *, + name: str | None = None, + project_id: str | None = None, + project_name: str | None = None, +) -> AgentStream | None: + """ + Retrieve an agent stream by name. + + Parameters + ---------- + name: + The agent stream name. + project_id: + The project ID (mutually exclusive with *project_name*). + project_name: + The project name (mutually exclusive with *project_id*). + + Returns + ------- + AgentStream | None + The agent stream if found, ``None`` otherwise. + """ + result = get_log_stream(name=name, project_id=project_id, project_name=project_name) + if result is None: + return None + stream = AgentStream.__new__(AgentStream) + stream.__dict__.update(result.__dict__) + return stream + + +def list_agent_streams( + *, + project_id: str | None = None, + project_name: str | None = None, + limit: Unset | int = 100, + starting_token: Unset | int = 0, +) -> builtins.list[AgentStream]: + """ + List agent streams for a project. + + Parameters + ---------- + project_id: + The project ID (mutually exclusive with *project_name*). + project_name: + The project name (mutually exclusive with *project_id*). + limit: + Maximum number of results per page. Defaults to 100. + starting_token: + Pagination token. Defaults to 0. + + Returns + ------- + list[AgentStream] + A page of agent streams. + """ + log_streams = list_log_streams( + project_id=project_id, + project_name=project_name, + limit=limit, + starting_token=starting_token, + ) + result: builtins.list[AgentStream] = [] + for ls in log_streams: + s = AgentStream.__new__(AgentStream) + s.__dict__.update(ls.__dict__) + result.append(s) + return result + + +def create_agent_stream( + name: str, + project_id: str | None = None, + project_name: str | None = None, +) -> AgentStream: + """ + Create a new agent stream. + + Parameters + ---------- + name: + The agent stream name. + project_id: + The project ID (mutually exclusive with *project_name*). + project_name: + The project name (mutually exclusive with *project_id*). + + Returns + ------- + AgentStream + The created agent stream. + """ + ls = create_log_stream(name=name, project_id=project_id, project_name=project_name) + stream = AgentStream.__new__(AgentStream) + stream.__dict__.update(ls.__dict__) + return stream + + +def enable_evaluators( + *, + agent_stream_name: str | None = None, + project_name: str | None = None, + metrics: builtins.list[SplunkAOMetrics | Metric | LocalMetricConfig | str], +) -> builtins.list[LocalMetricConfig]: + """ + Enable evaluators (formerly *metrics*) on an agent stream. + + Falls back to the ``SPLUNK_AO_LOG_STREAM`` and ``SPLUNK_AO_PROJECT`` + environment variables when *agent_stream_name* / *project_name* are + not provided explicitly. + + Parameters + ---------- + agent_stream_name: + The agent stream name. Falls back to ``SPLUNK_AO_LOG_STREAM`` env var. + project_name: + The project name. Falls back to ``SPLUNK_AO_PROJECT`` env var. + metrics: + Evaluators to enable. Accepts ``SplunkAOMetrics`` enum values, + ``Metric`` objects, ``LocalMetricConfig`` objects, or string names. + + Returns + ------- + list[LocalMetricConfig] + Local evaluator configurations that must be computed client-side. + """ + from splunk_ao.log_streams import LogStreams + + return LogStreams().enable_metrics( + log_stream_name=agent_stream_name, + project_name=project_name, + metrics=metrics, + ) + + +# --------------------------------------------------------------------------- +# Deprecated aliases for the old ``log_streams`` convenience functions +# --------------------------------------------------------------------------- + +def __getattr__(name: str): + _deprecated = { + "get_log_stream": ("get_agent_stream", get_agent_stream), + "list_log_streams": ("list_agent_streams", list_agent_streams), + "create_log_stream": ("create_agent_stream", create_agent_stream), + "enable_metrics": ("enable_evaluators", enable_evaluators), + } + if name in _deprecated: + new_name, obj = _deprecated[name] + warnings.warn( + f"splunk_ao.agent_streams.{name} is deprecated; use {new_name} instead.", + DeprecationWarning, + stacklevel=2, + ) + return obj + raise AttributeError(f"module 'splunk_ao.agent_streams' has no attribute {name!r}") diff --git a/src/splunk_ao/evaluator.py b/src/splunk_ao/evaluator.py new file mode 100644 index 00000000..b867c2a7 --- /dev/null +++ b/src/splunk_ao/evaluator.py @@ -0,0 +1,188 @@ +""" +Evaluators — the renamed successor to Metrics (HYBIM-730). + +``Evaluator`` and its concrete subclasses (``LlmEvaluator``, ``CodeEvaluator``, +``LocalEvaluator``, ``SplunkAOEvaluator``) are the canonical names going +forward. The old ``Metric``-prefixed names are kept as deprecated aliases in +``splunk_ao.__init__`` and will be removed in a future major release. + +The underlying implementation lives in ``splunk_ao.metric``; the API endpoints +still use the ``/scorers`` path (server-side rename is tracked separately). +""" +from __future__ import annotations + +import warnings + +from splunk_ao.metric import ( + BuiltInMetrics, + CodeMetric, + LlmMetric, + LocalMetric, + Metric, + SplunkAOMetric, +) + +__all__ = [ + "BuiltInEvaluators", + "CodeEvaluator", + "Evaluator", + "LlmEvaluator", + "LocalEvaluator", + "SplunkAOEvaluator", +] + + +class BuiltInEvaluators(BuiltInMetrics): + """ + Provides attribute-style access to built-in Splunk AO evaluators. + + This is the renamed successor to ``BuiltInMetrics``. Access built-in + evaluators via ``Evaluator.evaluators``. + + Examples + -------- + from splunk_ao import Evaluator + Evaluator.evaluators.correctness + Evaluator.evaluators.completeness + """ + + +class Evaluator(Metric): + """ + Base class for all Splunk AO evaluators. + + ``Evaluator`` is the new name for what was previously called a *Metric*. + All functionality is inherited from ``splunk_ao.metric.Metric`` unchanged. + + Use one of the concrete subclasses for new code: + + - ``SplunkAOEvaluator`` — built-in Splunk AO scorers (``Evaluator.evaluators.*``) + - ``LlmEvaluator`` — custom LLM-judge evaluators + - ``LocalEvaluator`` — local function-based evaluators + - ``CodeEvaluator`` — code-based evaluators + + Class Attributes + ---------------- + evaluators : BuiltInEvaluators + Access built-in Splunk AO evaluators. + + Examples + -------- + from splunk_ao import Evaluator, AgentStream, SplunkAOMetrics + + stream = AgentStream.get(name="prod-traces", project_name="my-project") + stream.set_metrics([ + Evaluator.evaluators.correctness, + Evaluator.evaluators.completeness, + ]) + + # Get an existing evaluator by name + ev = Evaluator.get(name="factuality-checker") + + # List all evaluators + evaluators = Evaluator.list() + """ + + evaluators = BuiltInEvaluators() + + # Keep ``metrics`` attribute as deprecated alias + @property # type: ignore[override] + def metrics(self): # type: ignore[override] + warnings.warn( + "Evaluator.metrics is deprecated; use Evaluator.evaluators instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.__class__.evaluators + + +class LlmEvaluator(LlmMetric): + """ + LLM-based evaluator — the renamed successor to ``LlmMetric``. + + See ``splunk_ao.metric.LlmMetric`` for the full API reference. + + Examples + -------- + from splunk_ao import LlmEvaluator + + ev = LlmEvaluator( + name="response_quality", + prompt="Rate the quality 1-10: {input} -> {output}", + model="gpt-4o-mini", + judges=3, + ).create() + """ + + +class CodeEvaluator(CodeMetric): + """ + Code-based evaluator — the renamed successor to ``CodeMetric``. + + See ``splunk_ao.metric.CodeMetric`` for the full API reference. + + Examples + -------- + from splunk_ao import CodeEvaluator + + ev = CodeEvaluator( + name="custom_scorer", + code="def scorer_fn(step): return 1.0", + ).create() + """ + + +class LocalEvaluator(LocalMetric): + """ + Local function-based evaluator — the renamed successor to ``LocalMetric``. + + See ``splunk_ao.metric.LocalMetric`` for the full API reference. + + Examples + -------- + from splunk_ao import LocalEvaluator + + def my_fn(trace): + return 0.5 + + ev = LocalEvaluator(name="my_scorer", scorer_fn=my_fn) + """ + + +class SplunkAOEvaluator(SplunkAOMetric): + """ + Built-in Splunk AO scorer evaluator — the renamed successor to ``SplunkAOMetric``. + + Access built-in evaluators via ``Evaluator.evaluators.*``. + + Examples + -------- + from splunk_ao import Evaluator + + ev = Evaluator.get(name="correctness") + assert isinstance(ev, SplunkAOEvaluator) + """ + + +# --------------------------------------------------------------------------- +# Deprecated aliases for the old Metric-prefixed names +# --------------------------------------------------------------------------- + +def __getattr__(name: str): + _deprecated = { + "Metric": ("Evaluator", Evaluator), + "LlmMetric": ("LlmEvaluator", LlmEvaluator), + "CodeMetric": ("CodeEvaluator", CodeEvaluator), + "LocalMetric": ("LocalEvaluator", LocalEvaluator), + "SplunkAOMetric": ("SplunkAOEvaluator", SplunkAOEvaluator), + "BuiltInMetrics": ("BuiltInEvaluators", BuiltInEvaluators), + } + if name in _deprecated: + new_name, obj = _deprecated[name] + warnings.warn( + f"splunk_ao.evaluator.{name} is deprecated; use {new_name} instead.", + DeprecationWarning, + stacklevel=2, + ) + return obj + raise AttributeError(f"module 'splunk_ao.evaluator' has no attribute {name!r}") diff --git a/src/splunk_ao/evaluators.py b/src/splunk_ao/evaluators.py new file mode 100644 index 00000000..25b6554d --- /dev/null +++ b/src/splunk_ao/evaluators.py @@ -0,0 +1,194 @@ +""" +Evaluators service layer — the renamed successor to Metrics (HYBIM-730). + +Provides ``Evaluators`` (service class) and the module-level convenience +functions ``create_custom_llm_evaluator``, ``get_evaluators``, and +``delete_evaluator``. + +The old ``metrics`` module and its symbols remain available but are deprecated. +""" +from __future__ import annotations + +import datetime +import warnings + +from splunk_ao.metrics import Metrics, create_custom_llm_metric, delete_metric, get_metrics +from splunk_ao.resources.models.base_scorer_version_response import BaseScorerVersionResponse +from splunk_ao.resources.models.output_type_enum import OutputTypeEnum +from splunk_ao.resources.models.log_records_metrics_response import LogRecordsMetricsResponse +from galileo_core.schemas.logging.step import StepType +from splunk_ao.search import FilterType + +__all__ = [ + "Evaluators", + "create_custom_llm_evaluator", + "delete_evaluator", + "get_evaluators", +] + + +class Evaluators(Metrics): + """ + Low-level service class for managing evaluators. + + ``Evaluators`` is the new name for the ``Metrics`` service class. + Inherits all methods from ``Metrics`` unchanged. + + Examples + -------- + from splunk_ao.evaluators import Evaluators + + svc = Evaluators() + svc.delete_metric(name="old-evaluator") + """ + + +# --------------------------------------------------------------------------- +# Module-level convenience functions +# --------------------------------------------------------------------------- + + +def create_custom_llm_evaluator( + name: str, + user_prompt: str, + node_level: StepType = StepType.llm, + cot_enabled: bool = True, + model_name: str = "gpt-4.1-mini", + num_judges: int = 3, + description: str = "", + tags: list[str] | None = None, + output_type: OutputTypeEnum = OutputTypeEnum.BOOLEAN, + ground_truth: bool = False, +) -> BaseScorerVersionResponse: + """ + Create a custom LLM evaluator. + + This is the renamed equivalent of ``create_custom_llm_metric`` from + ``splunk_ao.metrics``. + + Parameters + ---------- + name: + Name of the evaluator. + user_prompt: + Prompt template for the evaluator. + node_level: + Node level. Defaults to ``StepType.llm``. + cot_enabled: + Whether chain-of-thought reasoning is enabled. + model_name: + Model alias to use for judging. + num_judges: + Number of judge LLMs to use. + description: + Human-readable description. + tags: + Tags to associate with the evaluator. + output_type: + Output type (boolean, percentage, etc.). + ground_truth: + Whether the evaluator requires a ground-truth reference value. + + Returns + ------- + BaseScorerVersionResponse + The created evaluator version details. + """ + return create_custom_llm_metric( + name=name, + user_prompt=user_prompt, + node_level=node_level, + cot_enabled=cot_enabled, + model_name=model_name, + num_judges=num_judges, + description=description, + tags=tags, + output_type=output_type, + ground_truth=ground_truth, + ) + + +def get_evaluators( + project_id: str, + start_time: datetime.datetime, + end_time: datetime.datetime, + experiment_id: str | None = None, + agent_stream_id: str | None = None, + filters: list[FilterType] | None = None, + group_by: str | None = None, + interval: int = 5, +) -> LogRecordsMetricsResponse: + """ + Query evaluator results for a project. + + This is the renamed equivalent of ``get_metrics`` from ``splunk_ao.metrics``. + + Parameters + ---------- + project_id: + Project UUID. + start_time: + Start of the query window. + end_time: + End of the query window. + experiment_id: + Filter by experiment ID (optional). + agent_stream_id: + Filter by agent stream ID (optional). + filters: + Additional query filters. + group_by: + Field to group results by. + interval: + Time interval in seconds. + + Returns + ------- + LogRecordsMetricsResponse + Evaluator query results. + """ + return get_metrics( + project_id=project_id, + start_time=start_time, + end_time=end_time, + experiment_id=experiment_id, + log_stream_id=agent_stream_id, + filters=filters, + group_by=group_by, + interval=interval, + ) + + +def delete_evaluator(name: str) -> None: + """ + Delete an evaluator by name. + + This is the renamed equivalent of ``delete_metric`` from ``splunk_ao.metrics``. + + Parameters + ---------- + name: + The evaluator name to delete. + """ + return delete_metric(name=name) + + +# --------------------------------------------------------------------------- +# Deprecated aliases for old ``metrics`` module function names +# --------------------------------------------------------------------------- + +def __getattr__(name: str): + _deprecated = { + "create_custom_llm_metric": ("create_custom_llm_evaluator", create_custom_llm_evaluator), + "get_metrics": ("get_evaluators", get_evaluators), + "delete_metric": ("delete_evaluator", delete_evaluator), + } + if name in _deprecated: + new_name, obj = _deprecated[name] + warnings.warn( + f"splunk_ao.evaluators.{name} is deprecated; use {new_name} instead.", + DeprecationWarning, + stacklevel=2, + ) + return obj + raise AttributeError(f"module 'splunk_ao.evaluators' has no attribute {name!r}") diff --git a/src/splunk_ao/project.py b/src/splunk_ao/project.py index 05ca34dd..4b763d92 100644 --- a/src/splunk_ao/project.py +++ b/src/splunk_ao/project.py @@ -288,64 +288,114 @@ def list(cls) -> builtins.list[Project]: return [cls._from_api_response(retrieved_project) for retrieved_project in retrieved_projects] - def create_log_stream(self, name: str) -> LogStream: + def create_agent_stream(self, name: str) -> "AgentStream": """ - Create a new log stream for this project. + Create a new agent stream for this project. Args: - name (str): The name of the log stream to create. + name (str): The name of the agent stream to create. Returns ------- - LogStream: The created log stream. + AgentStream: The created agent stream. Examples -------- project = Project.get(name="My AI Project") - log_stream = project.create_log_stream(name="Production Logs") + stream = project.create_agent_stream(name="Production Traces") """ + from splunk_ao.agent_stream import AgentStream # lazy to avoid circular import + if self.id is None: - raise ValueError("Project ID is not set. Cannot create log stream for a local-only project.") + raise ValueError("Project ID is not set. Cannot create agent stream for a local-only project.") - # Use the LogStream pattern to avoid duplication - return LogStream(name=name, project_id=self.id).create() + return AgentStream(name=name, project_id=self.id).create() - def list_log_streams( + def create_log_stream(self, name: str) -> "AgentStream": + """ + Create a new agent stream for this project. + + .. deprecated:: + Use :meth:`create_agent_stream` instead. + + Args: + name (str): The name of the agent stream to create. + + Returns + ------- + AgentStream: The created agent stream. + + Examples + -------- + project = Project.get(name="My AI Project") + log_stream = project.create_log_stream(name="Production Logs") + """ + import warnings + warnings.warn( + "Project.create_log_stream() is deprecated; use Project.create_agent_stream() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.create_agent_stream(name=name) + + def list_agent_streams( self, *, limit: Unset | int = 100, starting_token: Unset | int = 0 - ) -> builtins.list[LogStream]: + ) -> "builtins.list[AgentStream]": """ - List log streams for this project. + List agent streams for this project. Returns a single page of results. Use `starting_token` (from `next_starting_token` on a prior response) to fetch subsequent pages. Args: - limit (Union[Unset, int]): Maximum number of log streams to return per page. Defaults to 100. - starting_token (Union[Unset, int]): Pagination token to start from. Defaults to 0 (first page). + limit (Union[Unset, int]): Maximum number of agent streams per page. Defaults to 100. + starting_token (Union[Unset, int]): Pagination token. Defaults to 0 (first page). Returns ------- - List[LogStream]: A page of log streams belonging to this project. + List[AgentStream]: A page of agent streams belonging to this project. Examples -------- project = Project.get(name="My AI Project") - log_streams = project.list_log_streams() - for stream in log_streams: - # Process each log stream + streams = project.list_agent_streams() + for stream in streams: pass + """ + from splunk_ao.agent_stream import AgentStream # lazy to avoid circular import - # Cap the number of returned log streams - log_streams = project.list_log_streams(limit=3) + if self.id is None: + raise ValueError("Project ID is not set. Cannot list agent streams for a local-only project.") - # Fetch the next page - page_2 = project.list_log_streams(starting_token=100) + return AgentStream.list(project_id=self.id, limit=limit, starting_token=starting_token) + + def list_log_streams( + self, *, limit: Unset | int = 100, starting_token: Unset | int = 0 + ) -> "builtins.list[AgentStream]": """ - if self.id is None: - raise ValueError("Project ID is not set. Cannot list log streams for a local-only project.") + List agent streams for this project. + + .. deprecated:: + Use :meth:`list_agent_streams` instead. + + Returns a single page of results. Use `starting_token` (from + `next_starting_token` on a prior response) to fetch subsequent pages. - # Use the LogStream pattern to avoid duplication - return LogStream.list(project_id=self.id, limit=limit, starting_token=starting_token) + Args: + limit (Union[Unset, int]): Maximum number of streams per page. Defaults to 100. + starting_token (Union[Unset, int]): Pagination token. Defaults to 0 (first page). + + Returns + ------- + List[AgentStream]: A page of agent streams belonging to this project. + """ + import warnings + warnings.warn( + "Project.list_log_streams() is deprecated; use Project.list_agent_streams() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.list_agent_streams(limit=limit, starting_token=starting_token) def list_experiments(self) -> builtins.list[Experiment]: """ @@ -414,24 +464,44 @@ def list_prompts(self) -> builtins.list[Prompt]: return Prompt.list(project_id=self.id) @property - def logstreams(self) -> builtins.list[LogStream]: + def agent_streams(self) -> "builtins.list[AgentStream]": """ - Property to access log streams for this project. + Property to access agent streams for this project. - This is a read-only property that returns the current list of log streams. - To create new log streams, use create_log_stream(). + This is a read-only property that returns the current list of agent streams. + To create new agent streams, use :meth:`create_agent_stream`. Returns ------- - List[LogStream]: A list of log streams belonging to this project. + List[AgentStream]: A list of agent streams belonging to this project. Examples -------- project = Project.get(name="My AI Project") - for stream in project.logstreams: + for stream in project.agent_streams: print(stream.name) """ - return self.list_log_streams() + return self.list_agent_streams() + + @property + def logstreams(self) -> "builtins.list[AgentStream]": + """ + Property to access agent streams for this project. + + .. deprecated:: + Use :attr:`agent_streams` instead. + + Returns + ------- + List[AgentStream]: A list of agent streams belonging to this project. + """ + import warnings + warnings.warn( + "Project.logstreams is deprecated; use Project.agent_streams instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.list_agent_streams() @property def experiments(self) -> builtins.list[Experiment]: @@ -895,3 +965,6 @@ def save(self) -> Project: from splunk_ao.experiment import Experiment # noqa: E402 from splunk_ao.log_stream import LogStream # noqa: E402 from splunk_ao.prompt import Prompt # noqa: E402 + +# AgentStream is imported lazily inside methods to avoid a secondary circular import: +# agent_stream → log_stream → project → agent_stream