From b30bc2f347af603a18a71aa5849283b165352825 Mon Sep 17 00:00:00 2001 From: "ci.datadog-api-spec" Date: Sun, 19 Jul 2026 15:51:25 +0000 Subject: [PATCH] Regenerate client from commit 7828438 of spec repo --- .generator/schemas/v2/openapi.yaml | 1737 ++++++++++++++--- docs/datadog_api_client.v2.model.rst | 238 +++ .../llm-observability/CreateLLMObsPrompt.py | 30 + .../CreateLLMObsPromptVersion.py | 35 + .../llm-observability/DeleteLLMObsPrompt.py | 20 + .../v2/llm-observability/GetLLMObsPrompt.py | 20 + .../GetLLMObsPromptVersion.py | 24 + .../ListLLMObsPromptVersions.py | 20 + .../v2/llm-observability/ListLLMObsPrompts.py | 14 + .../llm-observability/UpdateLLMObsPrompt.py | 31 + .../UpdateLLMObsPromptVersion.py | 39 + src/datadog_api_client/configuration.py | 9 + .../v2/api/llm_observability_api.py | 438 ++++- .../v2/model/llm_obs_create_prompt_data.py | 48 + .../llm_obs_create_prompt_data_attributes.py | 94 + .../v2/model/llm_obs_create_prompt_request.py | 40 + .../llm_obs_create_prompt_version_data.py | 52 + ...s_create_prompt_version_data_attributes.py | 79 + .../llm_obs_create_prompt_version_request.py | 40 + .../v2/model/llm_obs_deleted_prompt_data.py | 54 + .../llm_obs_deleted_prompt_data_attributes.py | 40 + .../model/llm_obs_deleted_prompt_response.py | 40 + .../v2/model/llm_obs_prompt_chat_message.py | 39 + .../v2/model/llm_obs_prompt_data.py | 54 + .../model/llm_obs_prompt_data_attributes.py | 165 ++ .../v2/model/llm_obs_prompt_dataset.py | 43 + .../v2/model/llm_obs_prompt_response.py | 40 + .../model/llm_obs_prompt_response_source.py | 38 + .../v2/model/llm_obs_prompt_sdk_data.py | 54 + .../llm_obs_prompt_sdk_data_attributes.py | 86 + .../v2/model/llm_obs_prompt_sdk_response.py | 40 + .../v2/model/llm_obs_prompt_template.py | 36 + .../v2/model/llm_obs_prompt_type.py | 35 + .../v2/model/llm_obs_prompt_version_data.py | 56 + .../llm_obs_prompt_version_data_attributes.py | 165 ++ .../model/llm_obs_prompt_version_list_data.py | 60 + ...obs_prompt_version_list_data_attributes.py | 155 ++ .../model/llm_obs_prompt_version_response.py | 40 + .../v2/model/llm_obs_prompt_version_type.py | 35 + .../model/llm_obs_prompt_versions_response.py | 40 + .../v2/model/llm_obs_prompts_response.py | 40 + .../v2/model/llm_obs_update_prompt_data.py | 48 + .../llm_obs_update_prompt_data_attributes.py | 43 + .../v2/model/llm_obs_update_prompt_request.py | 40 + .../llm_obs_update_prompt_version_data.py | 52 + ...s_update_prompt_version_data_attributes.py | 56 + .../llm_obs_update_prompt_version_request.py | 40 + src/datadog_api_client/v2/models/__init__.py | 74 + tests/v2/features/given.json | 28 + tests/v2/features/llm_observability.feature | 767 ++++++-- tests/v2/features/undo.json | 63 + 51 files changed, 5174 insertions(+), 400 deletions(-) create mode 100644 examples/v2/llm-observability/CreateLLMObsPrompt.py create mode 100644 examples/v2/llm-observability/CreateLLMObsPromptVersion.py create mode 100644 examples/v2/llm-observability/DeleteLLMObsPrompt.py create mode 100644 examples/v2/llm-observability/GetLLMObsPrompt.py create mode 100644 examples/v2/llm-observability/GetLLMObsPromptVersion.py create mode 100644 examples/v2/llm-observability/ListLLMObsPromptVersions.py create mode 100644 examples/v2/llm-observability/ListLLMObsPrompts.py create mode 100644 examples/v2/llm-observability/UpdateLLMObsPrompt.py create mode 100644 examples/v2/llm-observability/UpdateLLMObsPromptVersion.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_create_prompt_version_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_deleted_prompt_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_chat_message.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_dataset.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_response_source.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_sdk_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_template.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_type.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_version_type.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompt_versions_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_prompts_response.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_request.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data_attributes.py create mode 100644 src/datadog_api_client/v2/model/llm_obs_update_prompt_version_request.py diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index ed984a56d0..ce25f97bf7 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -1009,6 +1009,32 @@ components: required: true schema: type: string + LLMObsPromptIDPathParameter: + description: The customer-provided identifier of the LLM Observability prompt. + example: "my-prompt" + in: path + name: prompt_id + required: true + schema: + type: string + LLMObsPromptLabelQueryParameter: + description: >- + **Deprecated.** Optional label of the prompt version to return. Do not use this parameter for new integrations. If omitted, the latest version is returned. If the prompt has no labels, the latest version is returned even when a label is requested. + in: query + name: label + required: false + schema: + type: string + LLMObsPromptVersionPathParameter: + description: The version number of the LLM Observability prompt. + example: 1 + in: path + name: version + required: true + schema: + format: int64 + minimum: 1 + type: integer LinkIDPathParameter: description: "The UUID of the case link." in: path @@ -51818,6 +51844,106 @@ components: $ref: "#/components/schemas/LLMObsContentBlock" minItems: 1 type: array + LLMObsCreatePromptData: + description: Data object for creating an LLM Observability prompt. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCreatePromptDataAttributes" + type: + $ref: "#/components/schemas/LLMObsPromptType" + required: + - type + - attributes + type: object + LLMObsCreatePromptDataAttributes: + description: >- + Attributes for creating an LLM Observability prompt and its first version. `prompt_id` and `template` are required; all other attributes are optional. + properties: + description: + description: Optional description of the prompt. + type: string + env_ids: + description: Optional feature-flag environment UUIDs to enable for this prompt immediately after creation. + items: + type: string + type: array + labels: + deprecated: true + description: >- + Optional labels to attach to the first version. Do not use this attribute for new integrations. + items: + type: string + type: array + prompt_id: + description: Customer-provided identifier for the new prompt. + example: "my-prompt" + type: string + template: + $ref: "#/components/schemas/LLMObsPromptTemplate" + title: + description: Optional title of the prompt. + type: string + user_version: + description: Optional user-supplied version identifier for the first version. + type: string + required: + - prompt_id + - template + type: object + LLMObsCreatePromptRequest: + description: Request to create an LLM Observability prompt. + properties: + data: + $ref: "#/components/schemas/LLMObsCreatePromptData" + required: + - data + type: object + LLMObsCreatePromptVersionData: + description: Data object for creating an LLM Observability prompt version. + properties: + attributes: + $ref: "#/components/schemas/LLMObsCreatePromptVersionDataAttributes" + type: + $ref: "#/components/schemas/LLMObsPromptVersionType" + required: + - type + - attributes + type: object + LLMObsCreatePromptVersionDataAttributes: + description: >- + Attributes for creating a new version of an LLM Observability prompt. `template` is required; all other attributes are optional. + properties: + description: + description: Optional description of this version. + type: string + env_ids: + description: >- + Optional feature-flag environment UUIDs whose default variant should be set to this version after creation. + items: + type: string + type: array + labels: + deprecated: true + description: Optional labels to attach to this version. Do not use this attribute for new integrations. + items: + type: string + type: array + template: + $ref: "#/components/schemas/LLMObsPromptTemplate" + user_version: + description: Optional user-supplied version identifier for this version. + type: string + required: + - template + type: object + LLMObsCreatePromptVersionRequest: + description: Request to create a new version of an LLM Observability prompt. + properties: + data: + $ref: "#/components/schemas/LLMObsCreatePromptVersionData" + required: + - data + type: object LLMObsCursorMeta: description: Pagination cursor metadata. properties: @@ -53293,6 +53419,46 @@ components: required: - data type: object + LLMObsDeletedPromptData: + description: Data object confirming that an LLM Observability prompt was deleted. + properties: + attributes: + $ref: "#/components/schemas/LLMObsDeletedPromptDataAttributes" + id: + description: Unique identifier of the deleted prompt. + example: "" + type: string + type: + $ref: "#/components/schemas/LLMObsPromptType" + required: + - id + - type + - attributes + type: object + LLMObsDeletedPromptDataAttributes: + description: Attributes confirming that an LLM Observability prompt was deleted. + properties: + deleted_at: + description: Timestamp when the prompt was deleted. + example: "" + format: date-time + type: string + prompt_id: + description: Customer-provided identifier of the deleted prompt. + example: "" + type: string + required: + - prompt_id + - deleted_at + type: object + LLMObsDeletedPromptResponse: + description: Response confirming that an LLM Observability prompt was deleted. + properties: + data: + $ref: "#/components/schemas/LLMObsDeletedPromptData" + required: + - data + type: object LLMObsDisplayBlockAnnotatedInteractionItem: description: A display_block interaction with its associated annotations. properties: @@ -56387,6 +56553,425 @@ components: required: - data type: object + LLMObsPromptChatMessage: + description: A single chat message in a prompt template. + properties: + content: + description: Content of the message. + example: "" + type: string + role: + description: Role of the message (for example `system`, `user`, or `assistant`). + example: "" + type: string + required: + - role + - content + type: object + LLMObsPromptChatTemplate: + description: A chat prompt template. + items: + $ref: "#/components/schemas/LLMObsPromptChatMessage" + minItems: 1 + type: array + x-generate-alias-as-model: true + LLMObsPromptData: + description: Data object for an LLM Observability prompt. + properties: + attributes: + $ref: "#/components/schemas/LLMObsPromptDataAttributes" + id: + description: Unique identifier of the prompt. + example: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + type: string + type: + $ref: "#/components/schemas/LLMObsPromptType" + required: + - id + - type + - attributes + type: object + LLMObsPromptDataAttributes: + description: Attributes of an LLM Observability prompt registry entry. + properties: + author: + description: UUID of the user who authored the prompt. + type: string + created_at: + description: Timestamp when the prompt was created. + format: date-time + type: string + created_from: + description: >- + Source that created the prompt, such as `ui-registry`, `sdk-registry`, or `sdk-instrumentation`. + example: "" + type: string + datasets: + description: Datasets observed in runs associated with this prompt. + items: + $ref: "#/components/schemas/LLMObsPromptDataset" + type: array + description: + description: Description of the prompt. + type: string + extracted_from: + description: Source prompt from which this prompt was extracted, when applicable. + type: string + in_registry: + description: Whether the prompt is a registry entry (as opposed to a code-discovered prompt). + example: false + type: boolean + last_seen_at: + description: Timestamp of the most recent observed run of this prompt. + format: date-time + type: string + last_version_created_at: + description: Timestamp when the most recent version of the prompt was created. + format: date-time + type: string + ml_app: + description: The ML application this prompt is associated with. + type: string + ml_apps: + description: ML applications observed running this prompt. + items: + type: string + type: array + num_versions: + description: Number of versions of the prompt. + example: 0 + format: int64 + type: integer + prompt_id: + description: Customer-provided identifier of the prompt. + example: "my-prompt" + type: string + source: + $ref: "#/components/schemas/LLMObsPromptResponseSource" + tags: + description: Tags observed on runs of this prompt. + items: + type: string + type: array + title: + description: Title of the prompt. + type: string + required: + - prompt_id + - source + - num_versions + - in_registry + - created_from + type: object + LLMObsPromptDataset: + description: A dataset observed in runs associated with a prompt or prompt version. + properties: + id: + description: Unique identifier of the dataset. + example: "" + type: string + name: + description: Name of the dataset. + type: string + required: + - id + type: object + LLMObsPromptResponse: + description: Response containing a single LLM Observability prompt. + properties: + data: + $ref: "#/components/schemas/LLMObsPromptData" + required: + - data + type: object + LLMObsPromptResponseSource: + description: Whether the prompt was created from the registry or discovered from observed LLM calls. + enum: + - registry + - code + example: registry + type: string + x-enum-varnames: + - REGISTRY + - CODE + LLMObsPromptSDKData: + description: Data object for a flattened LLM Observability prompt version returned for SDK consumption. + properties: + attributes: + $ref: "#/components/schemas/LLMObsPromptSDKDataAttributes" + id: + description: Unique identifier of the prompt. + example: "" + type: string + type: + $ref: "#/components/schemas/LLMObsPromptType" + required: + - id + - type + - attributes + type: object + LLMObsPromptSDKDataAttributes: + description: >- + Attributes of a flattened prompt version returned for SDK consumption. Exactly one of `template` and `chat_template` is returned. + properties: + chat_template: + description: >- + Chat template for this prompt version, as a list of role/content messages. Omitted for text templates. + items: + $ref: "#/components/schemas/LLMObsPromptChatMessage" + type: array + label: + deprecated: true + description: >- + The label requested through the deprecated `label` query parameter. Empty if no label was requested. + type: string + prompt_id: + description: Customer-provided identifier of the prompt. + type: string + prompt_version_uuid: + description: Unique identifier of this prompt version. + type: string + template: + description: Text template for this prompt version. Omitted for chat templates. + type: string + version: + description: >- + Version identifier for this prompt version. This is the sequential version number unless a user-supplied version identifier was set, in which case that identifier is used instead. + type: string + type: object + LLMObsPromptSDKResponse: + description: Response containing a flattened LLM Observability prompt version for SDK consumption. + properties: + data: + $ref: "#/components/schemas/LLMObsPromptSDKData" + required: + - data + type: object + LLMObsPromptTemplate: + description: A text template or a list of chat messages. + example: L + oneOf: + - $ref: "#/components/schemas/LLMObsPromptTextTemplate" + - $ref: "#/components/schemas/LLMObsPromptChatTemplate" + LLMObsPromptTextTemplate: + description: A text prompt template. + minLength: 1 + type: string + LLMObsPromptType: + description: Resource type of an LLM Observability prompt. + enum: + - prompt-templates + example: prompt-templates + type: string + x-enum-varnames: + - PROMPT_TEMPLATES + LLMObsPromptVersionData: + description: Data object for a specific version of an LLM Observability prompt. + properties: + attributes: + $ref: "#/components/schemas/LLMObsPromptVersionDataAttributes" + id: + description: Unique identifier of the prompt version. + example: "" + type: string + type: + $ref: "#/components/schemas/LLMObsPromptVersionType" + required: + - id + - type + - attributes + type: object + LLMObsPromptVersionDataAttributes: + description: Attributes of a specific version of an LLM Observability prompt. + properties: + author: + description: UUID of the user who authored this version. + type: string + created_at: + description: Timestamp stored on this prompt version. + format: date-time + type: string + datasets: + description: Datasets observed in runs associated with this prompt version. + items: + $ref: "#/components/schemas/LLMObsPromptDataset" + type: array + description: + description: Description of this version. + type: string + labels: + deprecated: true + description: Labels attached to this version (for example `development`, `staging`, `production`). + items: + type: string + type: array + last_seen_at: + description: Timestamp of the most recent observed run of this prompt version. + format: date-time + type: string + ml_app: + description: The ML application this prompt is associated with. + type: string + ml_apps: + description: ML applications observed running this prompt version. + items: + type: string + type: array + prompt_id: + description: Customer-provided identifier of the parent prompt. + example: "" + type: string + prompt_uuid: + description: Unique identifier of the parent prompt. + example: "" + type: string + tags: + description: Tags observed on runs of this prompt version. + items: + type: string + type: array + template: + $ref: "#/components/schemas/LLMObsPromptTemplate" + user_version: + description: User-supplied identifier for this version. + type: string + version: + description: Sequential version number. + example: 1 + format: int64 + minimum: 1 + type: integer + version_created_at: + description: Timestamp when this version was created. + format: date-time + type: string + required: + - prompt_uuid + - prompt_id + - template + - version + type: object + LLMObsPromptVersionListData: + description: Data object for a prompt version returned in a list. + properties: + attributes: + $ref: "#/components/schemas/LLMObsPromptVersionListDataAttributes" + id: + description: Unique identifier of the prompt version. + example: "" + type: string + type: + $ref: "#/components/schemas/LLMObsPromptVersionType" + required: + - id + - type + - attributes + type: object + LLMObsPromptVersionListDataAttributes: + description: Attributes of a prompt version returned in a list, excluding its template. + properties: + author: + description: UUID of the user who authored this version. + type: string + created_at: + description: Timestamp stored on this prompt version. + format: date-time + type: string + datasets: + description: Datasets observed in runs associated with this prompt version. + items: + $ref: "#/components/schemas/LLMObsPromptDataset" + type: array + description: + description: Description of this version. + type: string + labels: + deprecated: true + description: Labels attached to this version (for example `development`, `staging`, `production`). + items: + type: string + type: array + last_seen_at: + description: Timestamp of the most recent observed run of this prompt version. + format: date-time + type: string + ml_app: + description: The ML application this prompt is associated with. + type: string + ml_apps: + description: ML applications observed running this prompt version. + items: + type: string + type: array + prompt_id: + description: Customer-provided identifier of the parent prompt. + example: "" + type: string + prompt_uuid: + description: Unique identifier of the parent prompt. + example: "" + type: string + tags: + description: Tags observed on runs of this prompt version. + items: + type: string + type: array + user_version: + description: User-supplied identifier for this version. + type: string + version: + description: Sequential version number. + example: 1 + format: int64 + minimum: 1 + type: integer + version_created_at: + description: Timestamp when this version was created. + format: date-time + type: string + required: + - prompt_uuid + - prompt_id + - version + type: object + LLMObsPromptVersionResponse: + description: Response containing a specific version of an LLM Observability prompt. + properties: + data: + $ref: "#/components/schemas/LLMObsPromptVersionData" + required: + - data + type: object + LLMObsPromptVersionType: + description: Resource type of an LLM Observability prompt version. + enum: + - prompt-template-versions + example: prompt-template-versions + type: string + x-enum-varnames: + - PROMPT_TEMPLATE_VERSIONS + LLMObsPromptVersionsResponse: + description: Response containing the versions of an LLM Observability prompt. + properties: + data: + description: Prompt versions ordered from newest to oldest. + items: + $ref: "#/components/schemas/LLMObsPromptVersionListData" + type: array + required: + - data + type: object + LLMObsPromptsResponse: + description: Response containing a list of LLM Observability prompts. + properties: + data: + description: List of LLM Observability prompts. + items: + $ref: "#/components/schemas/LLMObsPromptData" + type: array + required: + - data + type: object LLMObsRecordType: description: Resource type of LLM Observability dataset records. enum: @@ -56880,6 +57465,76 @@ components: - TRACE - EXPERIMENT_TRACE - SESSION + LLMObsUpdatePromptData: + description: Data object for updating an LLM Observability prompt. + properties: + attributes: + $ref: "#/components/schemas/LLMObsUpdatePromptDataAttributes" + type: + $ref: "#/components/schemas/LLMObsPromptType" + required: + - type + - attributes + type: object + LLMObsUpdatePromptDataAttributes: + description: >- + Attributes for updating an LLM Observability prompt. At least one of `title` or `description` must be provided; both attributes are optional individually. + minProperties: 1 + properties: + description: + description: Optional new description for the prompt. + type: string + title: + description: Optional new title for the prompt. + type: string + type: object + LLMObsUpdatePromptRequest: + description: Request to update an LLM Observability prompt's metadata. + properties: + data: + $ref: "#/components/schemas/LLMObsUpdatePromptData" + required: + - data + type: object + LLMObsUpdatePromptVersionData: + description: Data object for updating an LLM Observability prompt version. + properties: + attributes: + $ref: "#/components/schemas/LLMObsUpdatePromptVersionDataAttributes" + type: + $ref: "#/components/schemas/LLMObsPromptVersionType" + required: + - type + - attributes + type: object + LLMObsUpdatePromptVersionDataAttributes: + description: >- + Attributes for updating an LLM Observability prompt version. At least one of `description`, `labels`, or `env_ids` must be provided; all three attributes are optional individually. + minProperties: 1 + properties: + description: + description: Optional new description for this version. + type: string + env_ids: + description: Optional feature-flag environment UUIDs whose default variant should be set to this version. + items: + type: string + type: array + labels: + deprecated: true + description: Optional new labels for this version. Do not use this attribute for new integrations. + items: + type: string + type: array + type: object + LLMObsUpdatePromptVersionRequest: + description: Request to update an LLM Observability prompt version's metadata or feature-flag environments. + properties: + data: + $ref: "#/components/schemas/LLMObsUpdatePromptVersionData" + required: + - data + type: object LLMObsUpsertAnnotationItem: description: |- A single annotation to create or update. The annotation is matched by @@ -151107,18 +151762,693 @@ paths: security: - apiKeyAuth: [] appKeyAuth: [] - summary: Push events for an LLM Observability experiment + summary: Push events for an LLM Observability experiment + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/accounts: + get: + description: Retrieve the list of configured accounts for the specified LLM provider integration. + operationId: ListLLMObsIntegrationAccounts + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + - account_id: "org-XYZ123" + account_name: "Production OpenAI" + account_region: "" + id: "account-abc123" + integration: "openai" + schema: + items: + $ref: "#/components/schemas/LLMObsIntegrationAccount" + type: array + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM integration accounts + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/inference: + post: + description: Run an LLM inference request through the specified integration and account, returning the model response and token usage. + operationId: CreateLLMObsIntegrationInference + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" + requestBody: + content: + application/json: + examples: + default: + value: + max_tokens: 256 + messages: + - content: "What is the capital of France?" + role: "user" + model_id: "gpt-4o" + temperature: 0.7 + schema: + $ref: "#/components/schemas/LLMObsIntegrationInferenceRequest" + description: Inference request parameters. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + max_tokens: 256 + messages: + - content: "What is the capital of France?" + role: "user" + model_id: "gpt-4o" + response: + assessment: "pass" + content: "The capital of France is Paris." + finish_reason: "stop" + inference_codes: [] + input_tokens: 15 + latency: 843 + output_tokens: 9 + tools: [] + total_tokens: 24 + temperature: 0.7 + schema: + $ref: "#/components/schemas/LLMObsIntegrationInferenceResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + "500": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Internal Server Error + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Run an LLM inference + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/models: + get: + description: Retrieve the list of models available for the specified LLM provider integration and account. + operationId: ListLLMObsIntegrationModels + parameters: + - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + - has_access: true + id: "gpt-4o" + integration: "openai" + integration_display_name: "OpenAI" + json_schema: true + model_display_name: "GPT-4o" + model_id: "gpt-4o" + provider: "openai" + provider_display_name: "OpenAI" + schema: + items: + $ref: "#/components/schemas/LLMObsIntegrationModel" + type: array + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM integration models + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in Preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/projects: + get: + description: List all LLM Observability projects sorted by creation date, newest first. + operationId: ListLLMObsProjects + parameters: + - description: Filter projects by project ID. + in: query + name: filter[id] + schema: + type: string + - description: Filter projects by name. + in: query + name: filter[name] + schema: + type: string + - description: Use the Pagination cursor to retrieve the next page of results. + in: query + name: page[cursor] + schema: + type: string + - description: Maximum number of results to return per page. + in: query + name: page[limit] + schema: + format: int64 + type: integer + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + created_at: "2024-01-01T00:00:00+00:00" + description: "" + name: My LLM Project + updated_at: "2024-01-01T00:00:00+00:00" + id: 00000000-0000-0000-0000-000000000001 + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability projects + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + post: + description: Create a new LLM Observability project. Returns the existing project if a name conflict occurs. + operationId: CreateLLMObsProject + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + name: My LLM Project + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectRequest" + description: Create project payload. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + created_at: "2024-01-01T00:00:00+00:00" + description: "" + name: My LLM Project + updated_at: "2024-01-01T00:00:00+00:00" + id: 00000000-0000-0000-0000-000000000002 + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectResponse" + description: OK + "201": + content: + application/json: + examples: + default: + value: + data: + attributes: + created_at: "2024-01-01T00:00:00+00:00" + description: "" + name: My LLM Project + updated_at: "2024-01-01T00:00:00+00:00" + id: 00000000-0000-0000-0000-000000000003 + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectResponse" + description: Created + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Create an LLM Observability project + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/projects/delete: + post: + description: Delete one or more LLM Observability projects. + operationId: DeleteLLMObsProjects + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + project_ids: + - a33671aa-24fd-4dcd-9b33-a8ec7dde7751 + type: projects + schema: + $ref: "#/components/schemas/LLMObsDeleteProjectsRequest" + description: Delete projects payload. + required: true + responses: + "204": + description: No Content + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete LLM Observability projects + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/projects/{project_id}: + patch: + description: Partially update an existing LLM Observability project. + operationId: UpdateLLMObsProject + parameters: + - $ref: "#/components/parameters/LLMObsProjectIDPathParameter" + requestBody: + content: + application/json: + examples: + default: + value: + data: + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectUpdateRequest" + description: Update project payload. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + created_at: "2024-01-01T00:00:00+00:00" + description: "" + name: My LLM Project + updated_at: "2024-01-01T00:00:00+00:00" + id: 00000000-0000-0000-0000-000000000004 + type: projects + schema: + $ref: "#/components/schemas/LLMObsProjectResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Update an LLM Observability project + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/prompts: + get: + description: List all LLM Observability prompts in the prompt registry for the org. + operationId: ListLLMObsPrompts + parameters: + - description: Optional filter for prompts by prompt ID. + in: query + name: filter[prompt_id] + schema: + type: string + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + - attributes: + author: "b1883807-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2026-05-07T15:34:58.581041+00:00" + created_from: "ui-registry" + in_registry: true + last_version_created_at: "2026-05-07T15:34:58.581041+00:00" + num_versions: 1 + prompt_id: "genre_classifier" + source: registry + title: "Genre Classifier" + id: "015ac2ea-8a25-5f0f-946f-f48264d772eb" + type: prompt-templates + - attributes: + created_at: "2026-03-25T00:00:00+00:00" + created_from: "sdk-instrumentation" + in_registry: true + last_version_created_at: "2026-03-25T00:00:00+00:00" + ml_app: "dbm-assistant" + num_versions: 2 + prompt_id: "dbm-assistant-system-prompt" + source: registry + id: "bcd10c3d-8a25-5f0f-946f-f48264d772eb" + type: prompt-templates + schema: + $ref: "#/components/schemas/LLMObsPromptsResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: List LLM Observability prompts + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + post: + description: Create a new prompt (and its first version) in the LLM Observability prompt registry. + operationId: CreateLLMObsPrompt + requestBody: + content: + application/json: + examples: + default: + value: + data: + attributes: + prompt_id: "my-prompt" + template: + - content: "You are a helpful assistant." + role: "system" + - content: "Hello {{name}}" + role: "user" + title: "My prompt" + type: prompt-templates + schema: + $ref: "#/components/schemas/LLMObsCreatePromptRequest" + description: Create prompt payload. + required: true + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + author: "3b12f1df-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2025-01-01T00:00:00+00:00" + created_from: "sdk-registry" + in_registry: true + last_version_created_at: "2025-01-01T00:00:00+00:00" + num_versions: 1 + prompt_id: "my-prompt" + source: registry + title: "My prompt" + id: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + type: prompt-templates + schema: + $ref: "#/components/schemas/LLMObsPromptResponse" + description: OK + "400": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Bad Request + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "409": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Conflict + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Create an LLM Observability prompt + tags: + - LLM Observability + x-unstable: |- + **Note**: This endpoint is in preview and is subject to change. + If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). + /api/v2/llm-obs/v1/prompts/{prompt_id}: + delete: + description: >- + Soft-delete an LLM Observability prompt. The prompt's version rows are retained, but they are no longer accessible through the public prompt registry endpoints. + operationId: DeleteLLMObsPrompt + parameters: + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" + responses: + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + deleted_at: "2025-01-01T00:00:00.000000Z" + prompt_id: "my-prompt" + id: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + type: prompt-templates + schema: + $ref: "#/components/schemas/LLMObsDeletedPromptResponse" + description: OK + "401": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found + "429": + $ref: "#/components/responses/TooManyRequestsResponse" + security: + - apiKeyAuth: [] + appKeyAuth: [] + summary: Delete an LLM Observability prompt tags: - LLM Observability x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/integrations/{integration}/accounts: get: - description: Retrieve the list of configured accounts for the specified LLM provider integration. - operationId: ListLLMObsIntegrationAccounts + description: Get the latest version of an LLM Observability prompt by prompt ID. + operationId: GetLLMObsPrompt parameters: - - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" + - $ref: "#/components/parameters/LLMObsPromptLabelQueryParameter" responses: "200": content: @@ -151126,22 +152456,21 @@ paths: examples: default: value: - - account_id: "org-XYZ123" - account_name: "Production OpenAI" - account_region: "" - id: "account-abc123" - integration: "openai" + data: + attributes: + chat_template: + - content: "you are a test" + role: "system" + - content: "hello {{template_variable}}" + role: "user" + prompt_id: "my-prompt" + prompt_version_uuid: "20e5280b-c75d-5699-8a70-a2773a751428" + version: "1" + id: "78dbd07d-f964-5b1b-97aa-d711fa65b7f7" + type: prompt-templates schema: - items: - $ref: "#/components/schemas/LLMObsIntegrationAccount" - type: array + $ref: "#/components/schemas/LLMObsPromptSDKResponse" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Bad Request "401": content: application/json: @@ -151154,39 +152483,46 @@ paths: schema: $ref: "#/components/schemas/JSONAPIErrorResponse" description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found "429": $ref: "#/components/responses/TooManyRequestsResponse" security: - apiKeyAuth: [] appKeyAuth: [] - summary: List LLM integration accounts + summary: Get an LLM Observability prompt tags: - LLM Observability x-unstable: |- - **Note**: This endpoint is in Preview and is subject to change. + **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/inference: - post: - description: Run an LLM inference request through the specified integration and account, returning the model response and token usage. - operationId: CreateLLMObsIntegrationInference + patch: + description: |- + Update the title and/or description of an LLM Observability prompt. + + This endpoint does not recompute version aggregate data: `num_versions` and `last_version_created_at` + are not populated in the response and may show as `0` and a zero-value timestamp even if the prompt has + existing versions. + operationId: UpdateLLMObsPrompt parameters: - - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" - - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" requestBody: content: application/json: examples: default: value: - max_tokens: 256 - messages: - - content: "What is the capital of France?" - role: "user" - model_id: "gpt-4o" - temperature: 0.7 + data: + attributes: + title: "My prompt (updated)" + type: prompt-templates schema: - $ref: "#/components/schemas/LLMObsIntegrationInferenceRequest" - description: Inference request parameters. + $ref: "#/components/schemas/LLMObsUpdatePromptRequest" + description: Update prompt payload. required: true responses: "200": @@ -151195,24 +152531,22 @@ paths: examples: default: value: - max_tokens: 256 - messages: - - content: "What is the capital of France?" - role: "user" - model_id: "gpt-4o" - response: - assessment: "pass" - content: "The capital of France is Paris." - finish_reason: "stop" - inference_codes: [] - input_tokens: 15 - latency: 843 - output_tokens: 9 - tools: [] - total_tokens: 24 - temperature: 0.7 + data: + attributes: + author: "3b12f1df-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2025-01-01T00:00:00+00:00" + created_from: "ui-registry" + description: "My prompt description" + in_registry: true + last_version_created_at: "0001-01-01T00:00:00Z" + num_versions: 0 + prompt_id: "my-prompt" + source: registry + title: "My prompt (updated)" + id: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + type: prompt-templates schema: - $ref: "#/components/schemas/LLMObsIntegrationInferenceResponse" + $ref: "#/components/schemas/LLMObsPromptResponse" description: OK "400": content: @@ -151232,106 +152566,29 @@ paths: schema: $ref: "#/components/schemas/JSONAPIErrorResponse" description: Forbidden - "429": - $ref: "#/components/responses/TooManyRequestsResponse" - "500": - content: - application/json: - schema: - $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Internal Server Error - security: - - apiKeyAuth: [] - appKeyAuth: [] - summary: Run an LLM inference - tags: - - LLM Observability - x-unstable: |- - **Note**: This endpoint is in Preview and is subject to change. - If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/integrations/{integration}/{account_id}/models: - get: - description: Retrieve the list of models available for the specified LLM provider integration and account. - operationId: ListLLMObsIntegrationModels - parameters: - - $ref: "#/components/parameters/LLMObsIntegrationPathParameter" - - $ref: "#/components/parameters/LLMObsAccountIDPathParameter" - responses: - "200": - content: - application/json: - examples: - default: - value: - - has_access: true - id: "gpt-4o" - integration: "openai" - integration_display_name: "OpenAI" - json_schema: true - model_display_name: "GPT-4o" - model_id: "gpt-4o" - provider: "openai" - provider_display_name: "OpenAI" - schema: - items: - $ref: "#/components/schemas/LLMObsIntegrationModel" - type: array - description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Bad Request - "401": - content: - application/json: - schema: - $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Unauthorized - "403": + "404": content: application/json: schema: $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Forbidden + description: Not Found "429": $ref: "#/components/responses/TooManyRequestsResponse" security: - apiKeyAuth: [] appKeyAuth: [] - summary: List LLM integration models + summary: Update an LLM Observability prompt tags: - LLM Observability x-unstable: |- - **Note**: This endpoint is in Preview and is subject to change. + **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/projects: + /api/v2/llm-obs/v1/prompts/{prompt_id}/versions: get: - description: List all LLM Observability projects sorted by creation date, newest first. - operationId: ListLLMObsProjects + description: List all versions of an LLM Observability prompt, ordered newest to oldest. + operationId: ListLLMObsPromptVersions parameters: - - description: Filter projects by project ID. - in: query - name: filter[id] - schema: - type: string - - description: Filter projects by name. - in: query - name: filter[name] - schema: - type: string - - description: Use the Pagination cursor to retrieve the next page of results. - in: query - name: page[cursor] - schema: - type: string - - description: Maximum number of results to return per page. - in: query - name: page[limit] - schema: - format: int64 - type: integer + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" responses: "200": content: @@ -151341,21 +152598,25 @@ paths: value: data: - attributes: - created_at: "2024-01-01T00:00:00+00:00" - description: "" - name: My LLM Project - updated_at: "2024-01-01T00:00:00+00:00" - id: 00000000-0000-0000-0000-000000000001 - type: projects + author: "3b12f1df-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2025-01-01T00:00:00+00:00" + description: "My prompt version description" + last_seen_at: "2025-01-01T00:00:00+00:00" + ml_app: "my-ml-app" + ml_apps: + - "my-ml-app" + prompt_id: "my-prompt" + prompt_uuid: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + tags: + - "env:prod" + user_version: "v1" + version: 1 + version_created_at: "2025-01-01T00:00:00+00:00" + id: "9c6e5e2a-9b2a-4c6a-8f2a-1a2b3c4d5e6f" + type: prompt-template-versions schema: - $ref: "#/components/schemas/LLMObsProjectsResponse" + $ref: "#/components/schemas/LLMObsPromptVersionsResponse" description: OK - "400": - content: - application/json: - schema: - $ref: "#/components/schemas/JSONAPIErrorResponse" - description: Bad Request "401": content: application/json: @@ -151373,15 +152634,17 @@ paths: security: - apiKeyAuth: [] appKeyAuth: [] - summary: List LLM Observability projects + summary: List versions of an LLM Observability prompt tags: - LLM Observability x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). post: - description: Create a new LLM Observability project. Returns the existing project if a name conflict occurs. - operationId: CreateLLMObsProject + description: Create a new version of an existing LLM Observability prompt. + operationId: CreateLLMObsPromptVersion + parameters: + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" requestBody: content: application/json: @@ -151390,11 +152653,15 @@ paths: value: data: attributes: - name: My LLM Project - type: projects + template: + - content: "you are a test v2" + role: "system" + - content: "hello again {{template_variable}}" + role: "user" + type: prompt-template-versions schema: - $ref: "#/components/schemas/LLMObsProjectRequest" - description: Create project payload. + $ref: "#/components/schemas/LLMObsCreatePromptVersionRequest" + description: Create prompt version payload. required: true responses: "200": @@ -151405,32 +152672,22 @@ paths: value: data: attributes: - created_at: "2024-01-01T00:00:00+00:00" - description: "" - name: My LLM Project - updated_at: "2024-01-01T00:00:00+00:00" - id: 00000000-0000-0000-0000-000000000002 - type: projects + author: "c5e19e5a-ad6b-11ed-ab4f-927130d31ef9" + created_at: "2026-07-16T08:07:03.676127798Z" + prompt_id: "spec-audit-1784189220" + prompt_uuid: "78dbd07d-f964-5b1b-97aa-d711fa65b7f7" + template: + - content: "you are a test v2" + role: "system" + - content: "hello again {{template_variable}}" + role: "user" + version: 2 + version_created_at: "2026-07-16T08:07:03.676127798Z" + id: "d83ab666-61cc-5545-a83b-2424bb85467b" + type: prompt-template-versions schema: - $ref: "#/components/schemas/LLMObsProjectResponse" + $ref: "#/components/schemas/LLMObsPromptVersionResponse" description: OK - "201": - content: - application/json: - examples: - default: - value: - data: - attributes: - created_at: "2024-01-01T00:00:00+00:00" - description: "" - name: My LLM Project - updated_at: "2024-01-01T00:00:00+00:00" - id: 00000000-0000-0000-0000-000000000003 - type: projects - schema: - $ref: "#/components/schemas/LLMObsProjectResponse" - description: Created "400": content: application/json: @@ -151449,39 +152706,61 @@ paths: schema: $ref: "#/components/schemas/JSONAPIErrorResponse" description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found "429": $ref: "#/components/responses/TooManyRequestsResponse" security: - apiKeyAuth: [] appKeyAuth: [] - summary: Create an LLM Observability project + summary: Create a new LLM Observability prompt version tags: - LLM Observability x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/projects/delete: - post: - description: Delete one or more LLM Observability projects. - operationId: DeleteLLMObsProjects - requestBody: - content: - application/json: - examples: - default: - value: - data: - attributes: - project_ids: - - a33671aa-24fd-4dcd-9b33-a8ec7dde7751 - type: projects - schema: - $ref: "#/components/schemas/LLMObsDeleteProjectsRequest" - description: Delete projects payload. - required: true + /api/v2/llm-obs/v1/prompts/{prompt_id}/versions/{version}: + get: + description: Get the full template of a single, specific version of an LLM Observability prompt. + operationId: GetLLMObsPromptVersion + parameters: + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" + - $ref: "#/components/parameters/LLMObsPromptVersionPathParameter" responses: - "204": - description: No Content + "200": + content: + application/json: + examples: + default: + value: + data: + attributes: + author: "3b12f1df-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2025-01-01T00:00:00+00:00" + description: "My prompt version description" + last_seen_at: "2025-01-01T00:00:00+00:00" + ml_app: "my-ml-app" + ml_apps: + - "my-ml-app" + prompt_id: "my-prompt" + prompt_uuid: "4a1a28ff-8a25-5f0f-946f-f48264d772eb" + tags: + - "env:prod" + template: + - content: "Hello" + role: "user" + user_version: "v1" + version: 1 + version_created_at: "2025-01-01T00:00:00+00:00" + id: "9c6e5e2a-9b2a-4c6a-8f2a-1a2b3c4d5e6f" + type: prompt-template-versions + schema: + $ref: "#/components/schemas/LLMObsPromptVersionResponse" + description: OK "400": content: application/json: @@ -151500,23 +152779,30 @@ paths: schema: $ref: "#/components/schemas/JSONAPIErrorResponse" description: Forbidden + "404": + content: + application/json: + schema: + $ref: "#/components/schemas/JSONAPIErrorResponse" + description: Not Found "429": $ref: "#/components/responses/TooManyRequestsResponse" security: - apiKeyAuth: [] appKeyAuth: [] - summary: Delete LLM Observability projects + summary: Get a specific LLM Observability prompt version tags: - LLM Observability x-unstable: |- **Note**: This endpoint is in preview and is subject to change. If you have any feedback, contact [Datadog support](https://docs.datadoghq.com/help/). - /api/v2/llm-obs/v1/projects/{project_id}: patch: - description: Partially update an existing LLM Observability project. - operationId: UpdateLLMObsProject + description: >- + Update the description and/or feature-flag environments of a specific version of an LLM Observability prompt. + operationId: UpdateLLMObsPromptVersion parameters: - - $ref: "#/components/parameters/LLMObsProjectIDPathParameter" + - $ref: "#/components/parameters/LLMObsPromptIDPathParameter" + - $ref: "#/components/parameters/LLMObsPromptVersionPathParameter" requestBody: content: application/json: @@ -151524,10 +152810,12 @@ paths: default: value: data: - type: projects + attributes: + description: "Audit test version (updated)" + type: prompt-template-versions schema: - $ref: "#/components/schemas/LLMObsProjectUpdateRequest" - description: Update project payload. + $ref: "#/components/schemas/LLMObsUpdatePromptVersionRequest" + description: Update prompt version payload. required: true responses: "200": @@ -151538,14 +152826,22 @@ paths: value: data: attributes: - created_at: "2024-01-01T00:00:00+00:00" - description: "" - name: My LLM Project - updated_at: "2024-01-01T00:00:00+00:00" - id: 00000000-0000-0000-0000-000000000004 - type: projects + author: "3b12f1df-14fd-4e12-bd6f-4a2f5c8b3d1e" + created_at: "2025-01-01T00:00:00+00:00" + description: "Audit test version (updated)" + prompt_id: "my-prompt" + prompt_uuid: "78dbd07d-f964-5b1b-97aa-d711fa65b7f7" + template: + - content: "you are a test" + role: "system" + - content: "hello {{template_variable}}" + role: "user" + version: 1 + version_created_at: "2025-01-01T00:00:00+00:00" + id: "20e5280b-c75d-5699-8a70-a2773a751428" + type: prompt-template-versions schema: - $ref: "#/components/schemas/LLMObsProjectResponse" + $ref: "#/components/schemas/LLMObsPromptVersionResponse" description: OK "400": content: @@ -151576,7 +152872,7 @@ paths: security: - apiKeyAuth: [] appKeyAuth: [] - summary: Update an LLM Observability project + summary: Update a specific LLM Observability prompt version tags: - LLM Observability x-unstable: |- @@ -201942,7 +203238,8 @@ tags: description: Find out more at url: "https://docs.datadoghq.com/account_management/api-app-keys/" name: Key Management - - description: Manage LLM Observability spans, data, projects, datasets, dataset records, experiments, and annotations. + - description: >- + Manage LLM Observability spans, data, projects, datasets, dataset records, experiments, prompts, and annotations. name: LLM Observability - description: |- Search your logs and send them to your Datadog platform over HTTP. See the [Log Management page](https://docs.datadoghq.com/logs/) for more information. diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index 8ecba11fd3..d2607cf8f4 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -21956,6 +21956,48 @@ datadog\_api\_client.v2.model.llm\_obs\_content\_block\_type module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_data module +------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_data\_attributes module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_request module +---------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_version\_data module +---------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_version_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_version\_data\_attributes module +---------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_version_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_create\_prompt\_version\_request module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_create_prompt_version_request + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_cursor\_meta module ----------------------------------------------------------- @@ -22656,6 +22698,27 @@ datadog\_api\_client.v2.model.llm\_obs\_delete\_projects\_request module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_deleted\_prompt\_data module +-------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_deleted_prompt_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_deleted\_prompt\_data\_attributes module +-------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_deleted_prompt_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_deleted\_prompt\_response module +------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_deleted_prompt_response + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_display\_block\_annotated\_interaction\_item module ------------------------------------------------------------------------------------------- @@ -23727,6 +23790,139 @@ datadog\_api\_client.v2.model.llm\_obs\_projects\_response module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_prompt\_chat\_message module +-------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_chat_message + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_data module +----------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_data\_attributes module +----------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_dataset module +-------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_dataset + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_response module +--------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_response\_source module +----------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_response_source + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_sdk\_data module +---------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_sdk_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_sdk\_data\_attributes module +---------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_sdk_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_sdk\_response module +-------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_sdk_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_template module +--------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_template + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_type module +----------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_data module +-------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_data\_attributes module +-------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_list\_data module +-------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_list_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_list\_data\_attributes module +-------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_list_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_response module +------------------------------------------------------------------------ + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_version\_type module +-------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_version_type + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompt\_versions\_response module +------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompt_versions_response + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_prompts\_response module +---------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_prompts_response + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_record\_type module ----------------------------------------------------------- @@ -23902,6 +24098,48 @@ datadog\_api\_client.v2.model.llm\_obs\_trace\_interaction\_type module :members: :show-inheritance: +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_data module +------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_data\_attributes module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_request module +---------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_request + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_version\_data module +---------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_version_data + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_version\_data\_attributes module +---------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_version_data_attributes + :members: + :show-inheritance: + +datadog\_api\_client.v2.model.llm\_obs\_update\_prompt\_version\_request module +------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.llm_obs_update_prompt_version_request + :members: + :show-inheritance: + datadog\_api\_client.v2.model.llm\_obs\_upsert\_annotation\_item module ----------------------------------------------------------------------- diff --git a/examples/v2/llm-observability/CreateLLMObsPrompt.py b/examples/v2/llm-observability/CreateLLMObsPrompt.py new file mode 100644 index 0000000000..38d28838a3 --- /dev/null +++ b/examples/v2/llm-observability/CreateLLMObsPrompt.py @@ -0,0 +1,30 @@ +""" +Create an LLM Observability prompt returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_create_prompt_data import LLMObsCreatePromptData +from datadog_api_client.v2.model.llm_obs_create_prompt_data_attributes import LLMObsCreatePromptDataAttributes +from datadog_api_client.v2.model.llm_obs_create_prompt_request import LLMObsCreatePromptRequest +from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + +body = LLMObsCreatePromptRequest( + data=LLMObsCreatePromptData( + attributes=LLMObsCreatePromptDataAttributes( + env_ids=[], + labels=[], + prompt_id="Example-LLM-Observability", + template="Hello", + ), + type=LLMObsPromptType.PROMPT_TEMPLATES, + ), +) + +configuration = Configuration() +configuration.unstable_operations["create_llm_obs_prompt"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.create_llm_obs_prompt(body=body) + + print(response) diff --git a/examples/v2/llm-observability/CreateLLMObsPromptVersion.py b/examples/v2/llm-observability/CreateLLMObsPromptVersion.py new file mode 100644 index 0000000000..4eb11156ba --- /dev/null +++ b/examples/v2/llm-observability/CreateLLMObsPromptVersion.py @@ -0,0 +1,35 @@ +""" +Create a new LLM Observability prompt version returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_create_prompt_version_data import LLMObsCreatePromptVersionData +from datadog_api_client.v2.model.llm_obs_create_prompt_version_data_attributes import ( + LLMObsCreatePromptVersionDataAttributes, +) +from datadog_api_client.v2.model.llm_obs_create_prompt_version_request import LLMObsCreatePromptVersionRequest +from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +body = LLMObsCreatePromptVersionRequest( + data=LLMObsCreatePromptVersionData( + attributes=LLMObsCreatePromptVersionDataAttributes( + env_ids=[], + labels=[], + template="Hello v3", + ), + type=LLMObsPromptVersionType.PROMPT_TEMPLATE_VERSIONS, + ), +) + +configuration = Configuration() +configuration.unstable_operations["create_llm_obs_prompt_version"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.create_llm_obs_prompt_version(prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, body=body) + + print(response) diff --git a/examples/v2/llm-observability/DeleteLLMObsPrompt.py b/examples/v2/llm-observability/DeleteLLMObsPrompt.py new file mode 100644 index 0000000000..eee17bf17c --- /dev/null +++ b/examples/v2/llm-observability/DeleteLLMObsPrompt.py @@ -0,0 +1,20 @@ +""" +Delete an LLM Observability prompt returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +configuration = Configuration() +configuration.unstable_operations["delete_llm_obs_prompt"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.delete_llm_obs_prompt( + prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, + ) + + print(response) diff --git a/examples/v2/llm-observability/GetLLMObsPrompt.py b/examples/v2/llm-observability/GetLLMObsPrompt.py new file mode 100644 index 0000000000..ba64afd373 --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsPrompt.py @@ -0,0 +1,20 @@ +""" +Get an LLM Observability prompt returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +configuration = Configuration() +configuration.unstable_operations["get_llm_obs_prompt"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.get_llm_obs_prompt( + prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, + ) + + print(response) diff --git a/examples/v2/llm-observability/GetLLMObsPromptVersion.py b/examples/v2/llm-observability/GetLLMObsPromptVersion.py new file mode 100644 index 0000000000..ddbc5e96c4 --- /dev/null +++ b/examples/v2/llm-observability/GetLLMObsPromptVersion.py @@ -0,0 +1,24 @@ +""" +Get a specific LLM Observability prompt version returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +# there is a valid "prompt_version" in the system +PROMPT_VERSION_DATA_ATTRIBUTES_VERSION = environ["PROMPT_VERSION_DATA_ATTRIBUTES_VERSION"] + +configuration = Configuration() +configuration.unstable_operations["get_llm_obs_prompt_version"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.get_llm_obs_prompt_version( + prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, + version=int(PROMPT_VERSION_DATA_ATTRIBUTES_VERSION), + ) + + print(response) diff --git a/examples/v2/llm-observability/ListLLMObsPromptVersions.py b/examples/v2/llm-observability/ListLLMObsPromptVersions.py new file mode 100644 index 0000000000..56ad9fc80f --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsPromptVersions.py @@ -0,0 +1,20 @@ +""" +List versions of an LLM Observability prompt returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +configuration = Configuration() +configuration.unstable_operations["list_llm_obs_prompt_versions"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.list_llm_obs_prompt_versions( + prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, + ) + + print(response) diff --git a/examples/v2/llm-observability/ListLLMObsPrompts.py b/examples/v2/llm-observability/ListLLMObsPrompts.py new file mode 100644 index 0000000000..14491cb793 --- /dev/null +++ b/examples/v2/llm-observability/ListLLMObsPrompts.py @@ -0,0 +1,14 @@ +""" +List LLM Observability prompts returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi + +configuration = Configuration() +configuration.unstable_operations["list_llm_obs_prompts"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.list_llm_obs_prompts() + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsPrompt.py b/examples/v2/llm-observability/UpdateLLMObsPrompt.py new file mode 100644 index 0000000000..70721b7cc2 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsPrompt.py @@ -0,0 +1,31 @@ +""" +Update an LLM Observability prompt returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType +from datadog_api_client.v2.model.llm_obs_update_prompt_data import LLMObsUpdatePromptData +from datadog_api_client.v2.model.llm_obs_update_prompt_data_attributes import LLMObsUpdatePromptDataAttributes +from datadog_api_client.v2.model.llm_obs_update_prompt_request import LLMObsUpdatePromptRequest + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +body = LLMObsUpdatePromptRequest( + data=LLMObsUpdatePromptData( + attributes=LLMObsUpdatePromptDataAttributes( + title="New title", + ), + type=LLMObsPromptType.PROMPT_TEMPLATES, + ), +) + +configuration = Configuration() +configuration.unstable_operations["update_llm_obs_prompt"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.update_llm_obs_prompt(prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, body=body) + + print(response) diff --git a/examples/v2/llm-observability/UpdateLLMObsPromptVersion.py b/examples/v2/llm-observability/UpdateLLMObsPromptVersion.py new file mode 100644 index 0000000000..1297bc8124 --- /dev/null +++ b/examples/v2/llm-observability/UpdateLLMObsPromptVersion.py @@ -0,0 +1,39 @@ +""" +Update a specific LLM Observability prompt version returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.llm_observability_api import LLMObservabilityApi +from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType +from datadog_api_client.v2.model.llm_obs_update_prompt_version_data import LLMObsUpdatePromptVersionData +from datadog_api_client.v2.model.llm_obs_update_prompt_version_data_attributes import ( + LLMObsUpdatePromptVersionDataAttributes, +) +from datadog_api_client.v2.model.llm_obs_update_prompt_version_request import LLMObsUpdatePromptVersionRequest + +# there is a valid "prompt" in the system +PROMPT_DATA_ATTRIBUTES_PROMPT_ID = environ["PROMPT_DATA_ATTRIBUTES_PROMPT_ID"] + +# there is a valid "prompt_version" in the system +PROMPT_VERSION_DATA_ATTRIBUTES_VERSION = environ["PROMPT_VERSION_DATA_ATTRIBUTES_VERSION"] + +body = LLMObsUpdatePromptVersionRequest( + data=LLMObsUpdatePromptVersionData( + attributes=LLMObsUpdatePromptVersionDataAttributes( + env_ids=[], + labels=[], + ), + type=LLMObsPromptVersionType.PROMPT_TEMPLATE_VERSIONS, + ), +) + +configuration = Configuration() +configuration.unstable_operations["update_llm_obs_prompt_version"] = True +with ApiClient(configuration) as api_client: + api_instance = LLMObservabilityApi(api_client) + response = api_instance.update_llm_obs_prompt_version( + prompt_id=PROMPT_DATA_ATTRIBUTES_PROMPT_ID, version=int(PROMPT_VERSION_DATA_ATTRIBUTES_VERSION), body=body + ) + + print(response) diff --git a/src/datadog_api_client/configuration.py b/src/datadog_api_client/configuration.py index 6999357559..bda898da69 100644 --- a/src/datadog_api_client/configuration.py +++ b/src/datadog_api_client/configuration.py @@ -274,6 +274,8 @@ def __init__( "v2.create_llm_obs_experiment_events": False, "v2.create_llm_obs_integration_inference": False, "v2.create_llm_obs_project": False, + "v2.create_llm_obs_prompt": False, + "v2.create_llm_obs_prompt_version": False, "v2.delete_llm_obs_annotation_queue": False, "v2.delete_llm_obs_annotation_queue_interactions": False, "v2.delete_llm_obs_annotations": False, @@ -284,6 +286,7 @@ def __init__( "v2.delete_llm_obs_experiments": False, "v2.delete_llm_obs_patterns_config": False, "v2.delete_llm_obs_projects": False, + "v2.delete_llm_obs_prompt": False, "v2.export_llm_obs_dataset": False, "v2.get_llm_obs_annotated_interactions": False, "v2.get_llm_obs_annotated_interactions_by_trace_i_ds": False, @@ -292,6 +295,8 @@ def __init__( "v2.get_llm_obs_dataset_draft_state": False, "v2.get_llm_obs_patterns_config": False, "v2.get_llm_obs_patterns_run_status": False, + "v2.get_llm_obs_prompt": False, + "v2.get_llm_obs_prompt_version": False, "v2.list_llm_obs_annotation_queues": False, "v2.list_llm_obs_dataset_records": False, "v2.list_llm_obs_datasets": False, @@ -308,6 +313,8 @@ def __init__( "v2.list_llm_obs_patterns_topics": False, "v2.list_llm_obs_patterns_topics_with_clustered_points": False, "v2.list_llm_obs_projects": False, + "v2.list_llm_obs_prompts": False, + "v2.list_llm_obs_prompt_versions": False, "v2.list_llm_obs_spans": False, "v2.lock_llm_obs_dataset_draft_state": False, "v2.restore_llm_obs_dataset_version": False, @@ -323,6 +330,8 @@ def __init__( "v2.update_llm_obs_dataset_records": False, "v2.update_llm_obs_experiment": False, "v2.update_llm_obs_project": False, + "v2.update_llm_obs_prompt": False, + "v2.update_llm_obs_prompt_version": False, "v2.upload_llm_obs_dataset_records_file": False, "v2.upsert_llm_obs_annotations": False, "v2.upsert_llm_obs_patterns_config": False, diff --git a/src/datadog_api_client/v2/api/llm_observability_api.py b/src/datadog_api_client/v2/api/llm_observability_api.py index 8286556462..c526d31909 100644 --- a/src/datadog_api_client/v2/api/llm_observability_api.py +++ b/src/datadog_api_client/v2/api/llm_observability_api.py @@ -73,6 +73,16 @@ from datadog_api_client.v2.model.llm_obs_project_request import LLMObsProjectRequest from datadog_api_client.v2.model.llm_obs_delete_projects_request import LLMObsDeleteProjectsRequest from datadog_api_client.v2.model.llm_obs_project_update_request import LLMObsProjectUpdateRequest +from datadog_api_client.v2.model.llm_obs_prompts_response import LLMObsPromptsResponse +from datadog_api_client.v2.model.llm_obs_prompt_response import LLMObsPromptResponse +from datadog_api_client.v2.model.llm_obs_create_prompt_request import LLMObsCreatePromptRequest +from datadog_api_client.v2.model.llm_obs_deleted_prompt_response import LLMObsDeletedPromptResponse +from datadog_api_client.v2.model.llm_obs_prompt_sdk_response import LLMObsPromptSDKResponse +from datadog_api_client.v2.model.llm_obs_update_prompt_request import LLMObsUpdatePromptRequest +from datadog_api_client.v2.model.llm_obs_prompt_versions_response import LLMObsPromptVersionsResponse +from datadog_api_client.v2.model.llm_obs_prompt_version_response import LLMObsPromptVersionResponse +from datadog_api_client.v2.model.llm_obs_create_prompt_version_request import LLMObsCreatePromptVersionRequest +from datadog_api_client.v2.model.llm_obs_update_prompt_version_request import LLMObsUpdatePromptVersionRequest from datadog_api_client.v2.model.llm_obs_spans_response import LLMObsSpansResponse from datadog_api_client.v2.model.llm_obs_search_spans_request import LLMObsSearchSpansRequest from datadog_api_client.v2.model.llm_obs_patterns_clustered_points_response import LLMObsPatternsClusteredPointsResponse @@ -108,7 +118,7 @@ class LLMObservabilityApi: """ - Manage LLM Observability spans, data, projects, datasets, dataset records, experiments, and annotations. + Manage LLM Observability spans, data, projects, datasets, dataset records, experiments, prompts, and annotations. """ def __init__(self, api_client=None): @@ -402,6 +412,52 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._create_llm_obs_prompt_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts", + "operation_id": "create_llm_obs_prompt", + "http_method": "POST", + "version": "v2", + }, + params_map={ + "body": { + "required": True, + "openapi_types": (LLMObsCreatePromptRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + + self._create_llm_obs_prompt_version_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptVersionResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}/versions", + "operation_id": "create_llm_obs_prompt_version", + "http_method": "POST", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsCreatePromptVersionRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._delete_llm_obs_annotation_queue_endpoint = _Endpoint( settings={ "response_type": None, @@ -641,6 +697,29 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._delete_llm_obs_prompt_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsDeletedPromptResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}", + "operation_id": "delete_llm_obs_prompt", + "http_method": "DELETE", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._export_llm_obs_dataset_endpoint = _Endpoint( settings={ "response_type": (str,), @@ -862,6 +941,66 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._get_llm_obs_prompt_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptSDKResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}", + "operation_id": "get_llm_obs_prompt", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + "label": { + "openapi_types": (str,), + "attribute": "label", + "location": "query", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + + self._get_llm_obs_prompt_version_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptVersionResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}/versions/{version}", + "operation_id": "get_llm_obs_prompt_version", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + "version": { + "required": True, + "validation": { + "inclusive_minimum": 1, + }, + "openapi_types": (int,), + "attribute": "version", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._list_llm_obs_annotation_queues_endpoint = _Endpoint( settings={ "response_type": (LLMObsAnnotationQueuesResponse,), @@ -1387,6 +1526,51 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._list_llm_obs_prompts_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptsResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts", + "operation_id": "list_llm_obs_prompts", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "filter_prompt_id": { + "openapi_types": (str,), + "attribute": "filter[prompt_id]", + "location": "query", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + + self._list_llm_obs_prompt_versions_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptVersionsResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}/versions", + "operation_id": "list_llm_obs_prompt_versions", + "http_method": "GET", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + }, + headers_map={ + "accept": ["application/json"], + }, + api_client=api_client, + ) + self._list_llm_obs_spans_endpoint = _Endpoint( settings={ "response_type": (LLMObsSpansResponse,), @@ -1828,6 +2012,67 @@ def __init__(self, api_client=None): api_client=api_client, ) + self._update_llm_obs_prompt_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}", + "operation_id": "update_llm_obs_prompt", + "http_method": "PATCH", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsUpdatePromptRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + + self._update_llm_obs_prompt_version_endpoint = _Endpoint( + settings={ + "response_type": (LLMObsPromptVersionResponse,), + "auth": ["apiKeyAuth", "appKeyAuth"], + "endpoint_path": "/api/v2/llm-obs/v1/prompts/{prompt_id}/versions/{version}", + "operation_id": "update_llm_obs_prompt_version", + "http_method": "PATCH", + "version": "v2", + }, + params_map={ + "prompt_id": { + "required": True, + "openapi_types": (str,), + "attribute": "prompt_id", + "location": "path", + }, + "version": { + "required": True, + "validation": { + "inclusive_minimum": 1, + }, + "openapi_types": (int,), + "attribute": "version", + "location": "path", + }, + "body": { + "required": True, + "openapi_types": (LLMObsUpdatePromptVersionRequest,), + "location": "body", + }, + }, + headers_map={"accept": ["application/json"], "content_type": ["application/json"]}, + api_client=api_client, + ) + self._upload_llm_obs_dataset_records_file_endpoint = _Endpoint( settings={ "response_type": None, @@ -2184,6 +2429,45 @@ def create_llm_obs_project( return self._create_llm_obs_project_endpoint.call_with_http_info(**kwargs) + def create_llm_obs_prompt( + self, + body: LLMObsCreatePromptRequest, + ) -> LLMObsPromptResponse: + """Create an LLM Observability prompt. + + Create a new prompt (and its first version) in the LLM Observability prompt registry. + + :param body: Create prompt payload. + :type body: LLMObsCreatePromptRequest + :rtype: LLMObsPromptResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["body"] = body + + return self._create_llm_obs_prompt_endpoint.call_with_http_info(**kwargs) + + def create_llm_obs_prompt_version( + self, + prompt_id: str, + body: LLMObsCreatePromptVersionRequest, + ) -> LLMObsPromptVersionResponse: + """Create a new LLM Observability prompt version. + + Create a new version of an existing LLM Observability prompt. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :param body: Create prompt version payload. + :type body: LLMObsCreatePromptVersionRequest + :rtype: LLMObsPromptVersionResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + kwargs["body"] = body + + return self._create_llm_obs_prompt_version_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_annotation_queue( self, queue_id: str, @@ -2379,6 +2663,23 @@ def delete_llm_obs_projects( return self._delete_llm_obs_projects_endpoint.call_with_http_info(**kwargs) + def delete_llm_obs_prompt( + self, + prompt_id: str, + ) -> LLMObsDeletedPromptResponse: + """Delete an LLM Observability prompt. + + Soft-delete an LLM Observability prompt. The prompt's version rows are retained, but they are no longer accessible through the public prompt registry endpoints. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :rtype: LLMObsDeletedPromptResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + return self._delete_llm_obs_prompt_endpoint.call_with_http_info(**kwargs) + def export_llm_obs_dataset( self, project_id: str, @@ -2548,6 +2849,52 @@ def get_llm_obs_patterns_run_status( return self._get_llm_obs_patterns_run_status_endpoint.call_with_http_info(**kwargs) + def get_llm_obs_prompt( + self, + prompt_id: str, + *, + label: Union[str, UnsetType] = unset, + ) -> LLMObsPromptSDKResponse: + """Get an LLM Observability prompt. + + Get the latest version of an LLM Observability prompt by prompt ID. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :param label: **Deprecated.** Optional label of the prompt version to return. Do not use this parameter for new integrations. If omitted, the latest version is returned. If the prompt has no labels, the latest version is returned even when a label is requested. + :type label: str, optional + :rtype: LLMObsPromptSDKResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + if label is not unset: + kwargs["label"] = label + + return self._get_llm_obs_prompt_endpoint.call_with_http_info(**kwargs) + + def get_llm_obs_prompt_version( + self, + prompt_id: str, + version: int, + ) -> LLMObsPromptVersionResponse: + """Get a specific LLM Observability prompt version. + + Get the full template of a single, specific version of an LLM Observability prompt. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :param version: The version number of the LLM Observability prompt. + :type version: int + :rtype: LLMObsPromptVersionResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + kwargs["version"] = version + + return self._get_llm_obs_prompt_version_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_annotation_queues( self, *, @@ -3027,6 +3374,42 @@ def list_llm_obs_projects( return self._list_llm_obs_projects_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_prompts( + self, + *, + filter_prompt_id: Union[str, UnsetType] = unset, + ) -> LLMObsPromptsResponse: + """List LLM Observability prompts. + + List all LLM Observability prompts in the prompt registry for the org. + + :param filter_prompt_id: Optional filter for prompts by prompt ID. + :type filter_prompt_id: str, optional + :rtype: LLMObsPromptsResponse + """ + kwargs: Dict[str, Any] = {} + if filter_prompt_id is not unset: + kwargs["filter_prompt_id"] = filter_prompt_id + + return self._list_llm_obs_prompts_endpoint.call_with_http_info(**kwargs) + + def list_llm_obs_prompt_versions( + self, + prompt_id: str, + ) -> LLMObsPromptVersionsResponse: + """List versions of an LLM Observability prompt. + + List all versions of an LLM Observability prompt, ordered newest to oldest. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :rtype: LLMObsPromptVersionsResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + return self._list_llm_obs_prompt_versions_endpoint.call_with_http_info(**kwargs) + def list_llm_obs_spans( self, *, @@ -3425,6 +3808,59 @@ def update_llm_obs_project( return self._update_llm_obs_project_endpoint.call_with_http_info(**kwargs) + def update_llm_obs_prompt( + self, + prompt_id: str, + body: LLMObsUpdatePromptRequest, + ) -> LLMObsPromptResponse: + """Update an LLM Observability prompt. + + Update the title and/or description of an LLM Observability prompt. + + This endpoint does not recompute version aggregate data: ``num_versions`` and ``last_version_created_at`` + are not populated in the response and may show as ``0`` and a zero-value timestamp even if the prompt has + existing versions. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :param body: Update prompt payload. + :type body: LLMObsUpdatePromptRequest + :rtype: LLMObsPromptResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + kwargs["body"] = body + + return self._update_llm_obs_prompt_endpoint.call_with_http_info(**kwargs) + + def update_llm_obs_prompt_version( + self, + prompt_id: str, + version: int, + body: LLMObsUpdatePromptVersionRequest, + ) -> LLMObsPromptVersionResponse: + """Update a specific LLM Observability prompt version. + + Update the description and/or feature-flag environments of a specific version of an LLM Observability prompt. + + :param prompt_id: The customer-provided identifier of the LLM Observability prompt. + :type prompt_id: str + :param version: The version number of the LLM Observability prompt. + :type version: int + :param body: Update prompt version payload. + :type body: LLMObsUpdatePromptVersionRequest + :rtype: LLMObsPromptVersionResponse + """ + kwargs: Dict[str, Any] = {} + kwargs["prompt_id"] = prompt_id + + kwargs["version"] = version + + kwargs["body"] = body + + return self._update_llm_obs_prompt_version_endpoint.call_with_http_info(**kwargs) + def upload_llm_obs_dataset_records_file( self, project_id: str, diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_data.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_data.py new file mode 100644 index 0000000000..fed1ca531c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_data.py @@ -0,0 +1,48 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_create_prompt_data_attributes import LLMObsCreatePromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + +class LLMObsCreatePromptData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_create_prompt_data_attributes import LLMObsCreatePromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + return { + "attributes": (LLMObsCreatePromptDataAttributes,), + "type": (LLMObsPromptType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__(self_, attributes: LLMObsCreatePromptDataAttributes, type: LLMObsPromptType, **kwargs): + """ + Data object for creating an LLM Observability prompt. + + :param attributes: Attributes for creating an LLM Observability prompt and its first version. ``prompt_id`` and ``template`` are required; all other attributes are optional. + :type attributes: LLMObsCreatePromptDataAttributes + + :param type: Resource type of an LLM Observability prompt. + :type type: LLMObsPromptType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_data_attributes.py new file mode 100644 index 0000000000..fbfb299b4e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_data_attributes.py @@ -0,0 +1,94 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + +class LLMObsCreatePromptDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + + return { + "description": (str,), + "env_ids": ([str],), + "labels": ([str],), + "prompt_id": (str,), + "template": (LLMObsPromptTemplate,), + "title": (str,), + "user_version": (str,), + } + + attribute_map = { + "description": "description", + "env_ids": "env_ids", + "labels": "labels", + "prompt_id": "prompt_id", + "template": "template", + "title": "title", + "user_version": "user_version", + } + + def __init__( + self_, + prompt_id: str, + template: Union[LLMObsPromptTemplate, str, List[LLMObsPromptChatMessage]], + description: Union[str, UnsetType] = unset, + env_ids: Union[List[str], UnsetType] = unset, + labels: Union[List[str], UnsetType] = unset, + title: Union[str, UnsetType] = unset, + user_version: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + Attributes for creating an LLM Observability prompt and its first version. ``prompt_id`` and ``template`` are required; all other attributes are optional. + + :param description: Optional description of the prompt. + :type description: str, optional + + :param env_ids: Optional feature-flag environment UUIDs to enable for this prompt immediately after creation. + :type env_ids: [str], optional + + :param labels: Optional labels to attach to the first version. Do not use this attribute for new integrations. **Deprecated**. + :type labels: [str], optional + + :param prompt_id: Customer-provided identifier for the new prompt. + :type prompt_id: str + + :param template: A text template or a list of chat messages. + :type template: LLMObsPromptTemplate + + :param title: Optional title of the prompt. + :type title: str, optional + + :param user_version: Optional user-supplied version identifier for the first version. + :type user_version: str, optional + """ + if description is not unset: + kwargs["description"] = description + if env_ids is not unset: + kwargs["env_ids"] = env_ids + if labels is not unset: + kwargs["labels"] = labels + if title is not unset: + kwargs["title"] = title + if user_version is not unset: + kwargs["user_version"] = user_version + super().__init__(kwargs) + + self_.prompt_id = prompt_id + self_.template = template diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_request.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_request.py new file mode 100644 index 0000000000..a71473bdb9 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_create_prompt_data import LLMObsCreatePromptData + + +class LLMObsCreatePromptRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_create_prompt_data import LLMObsCreatePromptData + + return { + "data": (LLMObsCreatePromptData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsCreatePromptData, **kwargs): + """ + Request to create an LLM Observability prompt. + + :param data: Data object for creating an LLM Observability prompt. + :type data: LLMObsCreatePromptData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data.py new file mode 100644 index 0000000000..1c95c26fa7 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data.py @@ -0,0 +1,52 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_create_prompt_version_data_attributes import ( + LLMObsCreatePromptVersionDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + +class LLMObsCreatePromptVersionData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_create_prompt_version_data_attributes import ( + LLMObsCreatePromptVersionDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + return { + "attributes": (LLMObsCreatePromptVersionDataAttributes,), + "type": (LLMObsPromptVersionType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__(self_, attributes: LLMObsCreatePromptVersionDataAttributes, type: LLMObsPromptVersionType, **kwargs): + """ + Data object for creating an LLM Observability prompt version. + + :param attributes: Attributes for creating a new version of an LLM Observability prompt. ``template`` is required; all other attributes are optional. + :type attributes: LLMObsCreatePromptVersionDataAttributes + + :param type: Resource type of an LLM Observability prompt version. + :type type: LLMObsPromptVersionType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data_attributes.py new file mode 100644 index 0000000000..04a1589054 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_data_attributes.py @@ -0,0 +1,79 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + +class LLMObsCreatePromptVersionDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + + return { + "description": (str,), + "env_ids": ([str],), + "labels": ([str],), + "template": (LLMObsPromptTemplate,), + "user_version": (str,), + } + + attribute_map = { + "description": "description", + "env_ids": "env_ids", + "labels": "labels", + "template": "template", + "user_version": "user_version", + } + + def __init__( + self_, + template: Union[LLMObsPromptTemplate, str, List[LLMObsPromptChatMessage]], + description: Union[str, UnsetType] = unset, + env_ids: Union[List[str], UnsetType] = unset, + labels: Union[List[str], UnsetType] = unset, + user_version: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + Attributes for creating a new version of an LLM Observability prompt. ``template`` is required; all other attributes are optional. + + :param description: Optional description of this version. + :type description: str, optional + + :param env_ids: Optional feature-flag environment UUIDs whose default variant should be set to this version after creation. + :type env_ids: [str], optional + + :param labels: Optional labels to attach to this version. Do not use this attribute for new integrations. **Deprecated**. + :type labels: [str], optional + + :param template: A text template or a list of chat messages. + :type template: LLMObsPromptTemplate + + :param user_version: Optional user-supplied version identifier for this version. + :type user_version: str, optional + """ + if description is not unset: + kwargs["description"] = description + if env_ids is not unset: + kwargs["env_ids"] = env_ids + if labels is not unset: + kwargs["labels"] = labels + if user_version is not unset: + kwargs["user_version"] = user_version + super().__init__(kwargs) + + self_.template = template diff --git a/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_request.py b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_request.py new file mode 100644 index 0000000000..6d471c071e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_create_prompt_version_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_create_prompt_version_data import LLMObsCreatePromptVersionData + + +class LLMObsCreatePromptVersionRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_create_prompt_version_data import LLMObsCreatePromptVersionData + + return { + "data": (LLMObsCreatePromptVersionData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsCreatePromptVersionData, **kwargs): + """ + Request to create a new version of an LLM Observability prompt. + + :param data: Data object for creating an LLM Observability prompt version. + :type data: LLMObsCreatePromptVersionData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data.py b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data.py new file mode 100644 index 0000000000..12b790000e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_deleted_prompt_data_attributes import LLMObsDeletedPromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + +class LLMObsDeletedPromptData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_deleted_prompt_data_attributes import LLMObsDeletedPromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + return { + "attributes": (LLMObsDeletedPromptDataAttributes,), + "id": (str,), + "type": (LLMObsPromptType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__(self_, attributes: LLMObsDeletedPromptDataAttributes, id: str, type: LLMObsPromptType, **kwargs): + """ + Data object confirming that an LLM Observability prompt was deleted. + + :param attributes: Attributes confirming that an LLM Observability prompt was deleted. + :type attributes: LLMObsDeletedPromptDataAttributes + + :param id: Unique identifier of the deleted prompt. + :type id: str + + :param type: Resource type of an LLM Observability prompt. + :type type: LLMObsPromptType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data_attributes.py new file mode 100644 index 0000000000..9389b721e9 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_data_attributes.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, +) + + +class LLMObsDeletedPromptDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + return { + "deleted_at": (datetime,), + "prompt_id": (str,), + } + + attribute_map = { + "deleted_at": "deleted_at", + "prompt_id": "prompt_id", + } + + def __init__(self_, deleted_at: datetime, prompt_id: str, **kwargs): + """ + Attributes confirming that an LLM Observability prompt was deleted. + + :param deleted_at: Timestamp when the prompt was deleted. + :type deleted_at: datetime + + :param prompt_id: Customer-provided identifier of the deleted prompt. + :type prompt_id: str + """ + super().__init__(kwargs) + + self_.deleted_at = deleted_at + self_.prompt_id = prompt_id diff --git a/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_response.py b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_response.py new file mode 100644 index 0000000000..bfce71d0ca --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_deleted_prompt_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_deleted_prompt_data import LLMObsDeletedPromptData + + +class LLMObsDeletedPromptResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_deleted_prompt_data import LLMObsDeletedPromptData + + return { + "data": (LLMObsDeletedPromptData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsDeletedPromptData, **kwargs): + """ + Response confirming that an LLM Observability prompt was deleted. + + :param data: Data object confirming that an LLM Observability prompt was deleted. + :type data: LLMObsDeletedPromptData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_chat_message.py b/src/datadog_api_client/v2/model/llm_obs_prompt_chat_message.py new file mode 100644 index 0000000000..f3c5e151e3 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_chat_message.py @@ -0,0 +1,39 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +class LLMObsPromptChatMessage(ModelNormal): + @cached_property + def openapi_types(_): + return { + "content": (str,), + "role": (str,), + } + + attribute_map = { + "content": "content", + "role": "role", + } + + def __init__(self_, content: str, role: str, **kwargs): + """ + A single chat message in a prompt template. + + :param content: Content of the message. + :type content: str + + :param role: Role of the message (for example ``system`` , ``user`` , or ``assistant`` ). + :type role: str + """ + super().__init__(kwargs) + + self_.content = content + self_.role = role diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_data.py b/src/datadog_api_client/v2/model/llm_obs_prompt_data.py new file mode 100644 index 0000000000..87ca5e116a --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_data.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_data_attributes import LLMObsPromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + +class LLMObsPromptData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_data_attributes import LLMObsPromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + return { + "attributes": (LLMObsPromptDataAttributes,), + "id": (str,), + "type": (LLMObsPromptType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__(self_, attributes: LLMObsPromptDataAttributes, id: str, type: LLMObsPromptType, **kwargs): + """ + Data object for an LLM Observability prompt. + + :param attributes: Attributes of an LLM Observability prompt registry entry. + :type attributes: LLMObsPromptDataAttributes + + :param id: Unique identifier of the prompt. + :type id: str + + :param type: Resource type of an LLM Observability prompt. + :type type: LLMObsPromptType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_prompt_data_attributes.py new file mode 100644 index 0000000000..b3123da8df --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_data_attributes.py @@ -0,0 +1,165 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + from datadog_api_client.v2.model.llm_obs_prompt_response_source import LLMObsPromptResponseSource + + +class LLMObsPromptDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + from datadog_api_client.v2.model.llm_obs_prompt_response_source import LLMObsPromptResponseSource + + return { + "author": (str,), + "created_at": (datetime,), + "created_from": (str,), + "datasets": ([LLMObsPromptDataset],), + "description": (str,), + "extracted_from": (str,), + "in_registry": (bool,), + "last_seen_at": (datetime,), + "last_version_created_at": (datetime,), + "ml_app": (str,), + "ml_apps": ([str],), + "num_versions": (int,), + "prompt_id": (str,), + "source": (LLMObsPromptResponseSource,), + "tags": ([str],), + "title": (str,), + } + + attribute_map = { + "author": "author", + "created_at": "created_at", + "created_from": "created_from", + "datasets": "datasets", + "description": "description", + "extracted_from": "extracted_from", + "in_registry": "in_registry", + "last_seen_at": "last_seen_at", + "last_version_created_at": "last_version_created_at", + "ml_app": "ml_app", + "ml_apps": "ml_apps", + "num_versions": "num_versions", + "prompt_id": "prompt_id", + "source": "source", + "tags": "tags", + "title": "title", + } + + def __init__( + self_, + created_from: str, + in_registry: bool, + num_versions: int, + prompt_id: str, + source: LLMObsPromptResponseSource, + author: Union[str, UnsetType] = unset, + created_at: Union[datetime, UnsetType] = unset, + datasets: Union[List[LLMObsPromptDataset], UnsetType] = unset, + description: Union[str, UnsetType] = unset, + extracted_from: Union[str, UnsetType] = unset, + last_seen_at: Union[datetime, UnsetType] = unset, + last_version_created_at: Union[datetime, UnsetType] = unset, + ml_app: Union[str, UnsetType] = unset, + ml_apps: Union[List[str], UnsetType] = unset, + tags: Union[List[str], UnsetType] = unset, + title: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + Attributes of an LLM Observability prompt registry entry. + + :param author: UUID of the user who authored the prompt. + :type author: str, optional + + :param created_at: Timestamp when the prompt was created. + :type created_at: datetime, optional + + :param created_from: Source that created the prompt, such as ``ui-registry`` , ``sdk-registry`` , or ``sdk-instrumentation``. + :type created_from: str + + :param datasets: Datasets observed in runs associated with this prompt. + :type datasets: [LLMObsPromptDataset], optional + + :param description: Description of the prompt. + :type description: str, optional + + :param extracted_from: Source prompt from which this prompt was extracted, when applicable. + :type extracted_from: str, optional + + :param in_registry: Whether the prompt is a registry entry (as opposed to a code-discovered prompt). + :type in_registry: bool + + :param last_seen_at: Timestamp of the most recent observed run of this prompt. + :type last_seen_at: datetime, optional + + :param last_version_created_at: Timestamp when the most recent version of the prompt was created. + :type last_version_created_at: datetime, optional + + :param ml_app: The ML application this prompt is associated with. + :type ml_app: str, optional + + :param ml_apps: ML applications observed running this prompt. + :type ml_apps: [str], optional + + :param num_versions: Number of versions of the prompt. + :type num_versions: int + + :param prompt_id: Customer-provided identifier of the prompt. + :type prompt_id: str + + :param source: Whether the prompt was created from the registry or discovered from observed LLM calls. + :type source: LLMObsPromptResponseSource + + :param tags: Tags observed on runs of this prompt. + :type tags: [str], optional + + :param title: Title of the prompt. + :type title: str, optional + """ + if author is not unset: + kwargs["author"] = author + if created_at is not unset: + kwargs["created_at"] = created_at + if datasets is not unset: + kwargs["datasets"] = datasets + if description is not unset: + kwargs["description"] = description + if extracted_from is not unset: + kwargs["extracted_from"] = extracted_from + if last_seen_at is not unset: + kwargs["last_seen_at"] = last_seen_at + if last_version_created_at is not unset: + kwargs["last_version_created_at"] = last_version_created_at + if ml_app is not unset: + kwargs["ml_app"] = ml_app + if ml_apps is not unset: + kwargs["ml_apps"] = ml_apps + if tags is not unset: + kwargs["tags"] = tags + if title is not unset: + kwargs["title"] = title + super().__init__(kwargs) + + self_.created_from = created_from + self_.in_registry = in_registry + self_.num_versions = num_versions + self_.prompt_id = prompt_id + self_.source = source diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_dataset.py b/src/datadog_api_client/v2/model/llm_obs_prompt_dataset.py new file mode 100644 index 0000000000..096b10eb08 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_dataset.py @@ -0,0 +1,43 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsPromptDataset(ModelNormal): + @cached_property + def openapi_types(_): + return { + "id": (str,), + "name": (str,), + } + + attribute_map = { + "id": "id", + "name": "name", + } + + def __init__(self_, id: str, name: Union[str, UnsetType] = unset, **kwargs): + """ + A dataset observed in runs associated with a prompt or prompt version. + + :param id: Unique identifier of the dataset. + :type id: str + + :param name: Name of the dataset. + :type name: str, optional + """ + if name is not unset: + kwargs["name"] = name + super().__init__(kwargs) + + self_.id = id diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_response.py b/src/datadog_api_client/v2/model/llm_obs_prompt_response.py new file mode 100644 index 0000000000..90fd8692c6 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_data import LLMObsPromptData + + +class LLMObsPromptResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_data import LLMObsPromptData + + return { + "data": (LLMObsPromptData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsPromptData, **kwargs): + """ + Response containing a single LLM Observability prompt. + + :param data: Data object for an LLM Observability prompt. + :type data: LLMObsPromptData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_response_source.py b/src/datadog_api_client/v2/model/llm_obs_prompt_response_source.py new file mode 100644 index 0000000000..6da783e310 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_response_source.py @@ -0,0 +1,38 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsPromptResponseSource(ModelSimple): + """ + Whether the prompt was created from the registry or discovered from observed LLM calls. + + :param value: Must be one of ["registry", "code"]. + :type value: str + """ + + allowed_values = { + "registry", + "code", + } + REGISTRY: ClassVar["LLMObsPromptResponseSource"] + CODE: ClassVar["LLMObsPromptResponseSource"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsPromptResponseSource.REGISTRY = LLMObsPromptResponseSource("registry") +LLMObsPromptResponseSource.CODE = LLMObsPromptResponseSource("code") diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data.py b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data.py new file mode 100644 index 0000000000..06c0cd7519 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data.py @@ -0,0 +1,54 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_sdk_data_attributes import LLMObsPromptSDKDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + +class LLMObsPromptSDKData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_sdk_data_attributes import LLMObsPromptSDKDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + return { + "attributes": (LLMObsPromptSDKDataAttributes,), + "id": (str,), + "type": (LLMObsPromptType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__(self_, attributes: LLMObsPromptSDKDataAttributes, id: str, type: LLMObsPromptType, **kwargs): + """ + Data object for a flattened LLM Observability prompt version returned for SDK consumption. + + :param attributes: Attributes of a flattened prompt version returned for SDK consumption. Exactly one of ``template`` and ``chat_template`` is returned. + :type attributes: LLMObsPromptSDKDataAttributes + + :param id: Unique identifier of the prompt. + :type id: str + + :param type: Resource type of an LLM Observability prompt. + :type type: LLMObsPromptType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data_attributes.py new file mode 100644 index 0000000000..667c974fdb --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_data_attributes.py @@ -0,0 +1,86 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + +class LLMObsPromptSDKDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + return { + "chat_template": ([LLMObsPromptChatMessage],), + "label": (str,), + "prompt_id": (str,), + "prompt_version_uuid": (str,), + "template": (str,), + "version": (str,), + } + + attribute_map = { + "chat_template": "chat_template", + "label": "label", + "prompt_id": "prompt_id", + "prompt_version_uuid": "prompt_version_uuid", + "template": "template", + "version": "version", + } + + def __init__( + self_, + chat_template: Union[List[LLMObsPromptChatMessage], UnsetType] = unset, + label: Union[str, UnsetType] = unset, + prompt_id: Union[str, UnsetType] = unset, + prompt_version_uuid: Union[str, UnsetType] = unset, + template: Union[str, UnsetType] = unset, + version: Union[str, UnsetType] = unset, + **kwargs, + ): + """ + Attributes of a flattened prompt version returned for SDK consumption. Exactly one of ``template`` and ``chat_template`` is returned. + + :param chat_template: Chat template for this prompt version, as a list of role/content messages. Omitted for text templates. + :type chat_template: [LLMObsPromptChatMessage], optional + + :param label: The label requested through the deprecated ``label`` query parameter. Empty if no label was requested. **Deprecated**. + :type label: str, optional + + :param prompt_id: Customer-provided identifier of the prompt. + :type prompt_id: str, optional + + :param prompt_version_uuid: Unique identifier of this prompt version. + :type prompt_version_uuid: str, optional + + :param template: Text template for this prompt version. Omitted for chat templates. + :type template: str, optional + + :param version: Version identifier for this prompt version. This is the sequential version number unless a user-supplied version identifier was set, in which case that identifier is used instead. + :type version: str, optional + """ + if chat_template is not unset: + kwargs["chat_template"] = chat_template + if label is not unset: + kwargs["label"] = label + if prompt_id is not unset: + kwargs["prompt_id"] = prompt_id + if prompt_version_uuid is not unset: + kwargs["prompt_version_uuid"] = prompt_version_uuid + if template is not unset: + kwargs["template"] = template + if version is not unset: + kwargs["version"] = version + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_response.py b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_response.py new file mode 100644 index 0000000000..311953eefd --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_sdk_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_sdk_data import LLMObsPromptSDKData + + +class LLMObsPromptSDKResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_sdk_data import LLMObsPromptSDKData + + return { + "data": (LLMObsPromptSDKData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsPromptSDKData, **kwargs): + """ + Response containing a flattened LLM Observability prompt version for SDK consumption. + + :param data: Data object for a flattened LLM Observability prompt version returned for SDK consumption. + :type data: LLMObsPromptSDKData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_template.py b/src/datadog_api_client/v2/model/llm_obs_prompt_template.py new file mode 100644 index 0000000000..5d5340dc11 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_template.py @@ -0,0 +1,36 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelComposed, + cached_property, +) + + +class LLMObsPromptTemplate(ModelComposed): + def __init__(self, **kwargs): + """ + A text template or a list of chat messages. + """ + super().__init__(kwargs) + + @cached_property + def _composed_schemas(_): + # we need this here to make our import statements work + # we must store _composed_schemas in here so the code is only run + # when we invoke this method. If we kept this at the class + # level we would get an error because the class level + # code would be run when this module is imported, and these composed + # classes don't exist yet because their module has not finished + # loading + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + return { + "oneOf": [ + str, + [LLMObsPromptChatMessage], + ], + } diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_type.py b/src/datadog_api_client/v2/model/llm_obs_prompt_type.py new file mode 100644 index 0000000000..ec0d5b0f89 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsPromptType(ModelSimple): + """ + Resource type of an LLM Observability prompt. + + :param value: If omitted defaults to "prompt-templates". Must be one of ["prompt-templates"]. + :type value: str + """ + + allowed_values = { + "prompt-templates", + } + PROMPT_TEMPLATES: ClassVar["LLMObsPromptType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsPromptType.PROMPT_TEMPLATES = LLMObsPromptType("prompt-templates") diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_data.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_data.py new file mode 100644 index 0000000000..3068f7f89d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_data.py @@ -0,0 +1,56 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_version_data_attributes import LLMObsPromptVersionDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + +class LLMObsPromptVersionData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_version_data_attributes import LLMObsPromptVersionDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + return { + "attributes": (LLMObsPromptVersionDataAttributes,), + "id": (str,), + "type": (LLMObsPromptVersionType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsPromptVersionDataAttributes, id: str, type: LLMObsPromptVersionType, **kwargs + ): + """ + Data object for a specific version of an LLM Observability prompt. + + :param attributes: Attributes of a specific version of an LLM Observability prompt. + :type attributes: LLMObsPromptVersionDataAttributes + + :param id: Unique identifier of the prompt version. + :type id: str + + :param type: Resource type of an LLM Observability prompt version. + :type type: LLMObsPromptVersionType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_data_attributes.py new file mode 100644 index 0000000000..97256f812d --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_data_attributes.py @@ -0,0 +1,165 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage + + +class LLMObsPromptVersionDataAttributes(ModelNormal): + validations = { + "version": { + "inclusive_minimum": 1, + }, + } + + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate + + return { + "author": (str,), + "created_at": (datetime,), + "datasets": ([LLMObsPromptDataset],), + "description": (str,), + "labels": ([str],), + "last_seen_at": (datetime,), + "ml_app": (str,), + "ml_apps": ([str],), + "prompt_id": (str,), + "prompt_uuid": (str,), + "tags": ([str],), + "template": (LLMObsPromptTemplate,), + "user_version": (str,), + "version": (int,), + "version_created_at": (datetime,), + } + + attribute_map = { + "author": "author", + "created_at": "created_at", + "datasets": "datasets", + "description": "description", + "labels": "labels", + "last_seen_at": "last_seen_at", + "ml_app": "ml_app", + "ml_apps": "ml_apps", + "prompt_id": "prompt_id", + "prompt_uuid": "prompt_uuid", + "tags": "tags", + "template": "template", + "user_version": "user_version", + "version": "version", + "version_created_at": "version_created_at", + } + + def __init__( + self_, + prompt_id: str, + prompt_uuid: str, + template: Union[LLMObsPromptTemplate, str, List[LLMObsPromptChatMessage]], + version: int, + author: Union[str, UnsetType] = unset, + created_at: Union[datetime, UnsetType] = unset, + datasets: Union[List[LLMObsPromptDataset], UnsetType] = unset, + description: Union[str, UnsetType] = unset, + labels: Union[List[str], UnsetType] = unset, + last_seen_at: Union[datetime, UnsetType] = unset, + ml_app: Union[str, UnsetType] = unset, + ml_apps: Union[List[str], UnsetType] = unset, + tags: Union[List[str], UnsetType] = unset, + user_version: Union[str, UnsetType] = unset, + version_created_at: Union[datetime, UnsetType] = unset, + **kwargs, + ): + """ + Attributes of a specific version of an LLM Observability prompt. + + :param author: UUID of the user who authored this version. + :type author: str, optional + + :param created_at: Timestamp stored on this prompt version. + :type created_at: datetime, optional + + :param datasets: Datasets observed in runs associated with this prompt version. + :type datasets: [LLMObsPromptDataset], optional + + :param description: Description of this version. + :type description: str, optional + + :param labels: Labels attached to this version (for example ``development`` , ``staging`` , ``production`` ). **Deprecated**. + :type labels: [str], optional + + :param last_seen_at: Timestamp of the most recent observed run of this prompt version. + :type last_seen_at: datetime, optional + + :param ml_app: The ML application this prompt is associated with. + :type ml_app: str, optional + + :param ml_apps: ML applications observed running this prompt version. + :type ml_apps: [str], optional + + :param prompt_id: Customer-provided identifier of the parent prompt. + :type prompt_id: str + + :param prompt_uuid: Unique identifier of the parent prompt. + :type prompt_uuid: str + + :param tags: Tags observed on runs of this prompt version. + :type tags: [str], optional + + :param template: A text template or a list of chat messages. + :type template: LLMObsPromptTemplate + + :param user_version: User-supplied identifier for this version. + :type user_version: str, optional + + :param version: Sequential version number. + :type version: int + + :param version_created_at: Timestamp when this version was created. + :type version_created_at: datetime, optional + """ + if author is not unset: + kwargs["author"] = author + if created_at is not unset: + kwargs["created_at"] = created_at + if datasets is not unset: + kwargs["datasets"] = datasets + if description is not unset: + kwargs["description"] = description + if labels is not unset: + kwargs["labels"] = labels + if last_seen_at is not unset: + kwargs["last_seen_at"] = last_seen_at + if ml_app is not unset: + kwargs["ml_app"] = ml_app + if ml_apps is not unset: + kwargs["ml_apps"] = ml_apps + if tags is not unset: + kwargs["tags"] = tags + if user_version is not unset: + kwargs["user_version"] = user_version + if version_created_at is not unset: + kwargs["version_created_at"] = version_created_at + super().__init__(kwargs) + + self_.prompt_id = prompt_id + self_.prompt_uuid = prompt_uuid + self_.template = template + self_.version = version diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data.py new file mode 100644 index 0000000000..0b8507c6fd --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data.py @@ -0,0 +1,60 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_version_list_data_attributes import ( + LLMObsPromptVersionListDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + +class LLMObsPromptVersionListData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_version_list_data_attributes import ( + LLMObsPromptVersionListDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + return { + "attributes": (LLMObsPromptVersionListDataAttributes,), + "id": (str,), + "type": (LLMObsPromptVersionType,), + } + + attribute_map = { + "attributes": "attributes", + "id": "id", + "type": "type", + } + + def __init__( + self_, attributes: LLMObsPromptVersionListDataAttributes, id: str, type: LLMObsPromptVersionType, **kwargs + ): + """ + Data object for a prompt version returned in a list. + + :param attributes: Attributes of a prompt version returned in a list, excluding its template. + :type attributes: LLMObsPromptVersionListDataAttributes + + :param id: Unique identifier of the prompt version. + :type id: str + + :param type: Resource type of an LLM Observability prompt version. + :type type: LLMObsPromptVersionType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.id = id + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data_attributes.py new file mode 100644 index 0000000000..610cf6daa8 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_list_data_attributes.py @@ -0,0 +1,155 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + datetime, + unset, + UnsetType, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + + +class LLMObsPromptVersionListDataAttributes(ModelNormal): + validations = { + "version": { + "inclusive_minimum": 1, + }, + } + + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset + + return { + "author": (str,), + "created_at": (datetime,), + "datasets": ([LLMObsPromptDataset],), + "description": (str,), + "labels": ([str],), + "last_seen_at": (datetime,), + "ml_app": (str,), + "ml_apps": ([str],), + "prompt_id": (str,), + "prompt_uuid": (str,), + "tags": ([str],), + "user_version": (str,), + "version": (int,), + "version_created_at": (datetime,), + } + + attribute_map = { + "author": "author", + "created_at": "created_at", + "datasets": "datasets", + "description": "description", + "labels": "labels", + "last_seen_at": "last_seen_at", + "ml_app": "ml_app", + "ml_apps": "ml_apps", + "prompt_id": "prompt_id", + "prompt_uuid": "prompt_uuid", + "tags": "tags", + "user_version": "user_version", + "version": "version", + "version_created_at": "version_created_at", + } + + def __init__( + self_, + prompt_id: str, + prompt_uuid: str, + version: int, + author: Union[str, UnsetType] = unset, + created_at: Union[datetime, UnsetType] = unset, + datasets: Union[List[LLMObsPromptDataset], UnsetType] = unset, + description: Union[str, UnsetType] = unset, + labels: Union[List[str], UnsetType] = unset, + last_seen_at: Union[datetime, UnsetType] = unset, + ml_app: Union[str, UnsetType] = unset, + ml_apps: Union[List[str], UnsetType] = unset, + tags: Union[List[str], UnsetType] = unset, + user_version: Union[str, UnsetType] = unset, + version_created_at: Union[datetime, UnsetType] = unset, + **kwargs, + ): + """ + Attributes of a prompt version returned in a list, excluding its template. + + :param author: UUID of the user who authored this version. + :type author: str, optional + + :param created_at: Timestamp stored on this prompt version. + :type created_at: datetime, optional + + :param datasets: Datasets observed in runs associated with this prompt version. + :type datasets: [LLMObsPromptDataset], optional + + :param description: Description of this version. + :type description: str, optional + + :param labels: Labels attached to this version (for example ``development`` , ``staging`` , ``production`` ). **Deprecated**. + :type labels: [str], optional + + :param last_seen_at: Timestamp of the most recent observed run of this prompt version. + :type last_seen_at: datetime, optional + + :param ml_app: The ML application this prompt is associated with. + :type ml_app: str, optional + + :param ml_apps: ML applications observed running this prompt version. + :type ml_apps: [str], optional + + :param prompt_id: Customer-provided identifier of the parent prompt. + :type prompt_id: str + + :param prompt_uuid: Unique identifier of the parent prompt. + :type prompt_uuid: str + + :param tags: Tags observed on runs of this prompt version. + :type tags: [str], optional + + :param user_version: User-supplied identifier for this version. + :type user_version: str, optional + + :param version: Sequential version number. + :type version: int + + :param version_created_at: Timestamp when this version was created. + :type version_created_at: datetime, optional + """ + if author is not unset: + kwargs["author"] = author + if created_at is not unset: + kwargs["created_at"] = created_at + if datasets is not unset: + kwargs["datasets"] = datasets + if description is not unset: + kwargs["description"] = description + if labels is not unset: + kwargs["labels"] = labels + if last_seen_at is not unset: + kwargs["last_seen_at"] = last_seen_at + if ml_app is not unset: + kwargs["ml_app"] = ml_app + if ml_apps is not unset: + kwargs["ml_apps"] = ml_apps + if tags is not unset: + kwargs["tags"] = tags + if user_version is not unset: + kwargs["user_version"] = user_version + if version_created_at is not unset: + kwargs["version_created_at"] = version_created_at + super().__init__(kwargs) + + self_.prompt_id = prompt_id + self_.prompt_uuid = prompt_uuid + self_.version = version diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_response.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_response.py new file mode 100644 index 0000000000..e248441fb3 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_version_data import LLMObsPromptVersionData + + +class LLMObsPromptVersionResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_version_data import LLMObsPromptVersionData + + return { + "data": (LLMObsPromptVersionData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsPromptVersionData, **kwargs): + """ + Response containing a specific version of an LLM Observability prompt. + + :param data: Data object for a specific version of an LLM Observability prompt. + :type data: LLMObsPromptVersionData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_version_type.py b/src/datadog_api_client/v2/model/llm_obs_prompt_version_type.py new file mode 100644 index 0000000000..e0a999571c --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_version_type.py @@ -0,0 +1,35 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class LLMObsPromptVersionType(ModelSimple): + """ + Resource type of an LLM Observability prompt version. + + :param value: If omitted defaults to "prompt-template-versions". Must be one of ["prompt-template-versions"]. + :type value: str + """ + + allowed_values = { + "prompt-template-versions", + } + PROMPT_TEMPLATE_VERSIONS: ClassVar["LLMObsPromptVersionType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +LLMObsPromptVersionType.PROMPT_TEMPLATE_VERSIONS = LLMObsPromptVersionType("prompt-template-versions") diff --git a/src/datadog_api_client/v2/model/llm_obs_prompt_versions_response.py b/src/datadog_api_client/v2/model/llm_obs_prompt_versions_response.py new file mode 100644 index 0000000000..d94c0efec3 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompt_versions_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_version_list_data import LLMObsPromptVersionListData + + +class LLMObsPromptVersionsResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_version_list_data import LLMObsPromptVersionListData + + return { + "data": ([LLMObsPromptVersionListData],), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: List[LLMObsPromptVersionListData], **kwargs): + """ + Response containing the versions of an LLM Observability prompt. + + :param data: Prompt versions ordered from newest to oldest. + :type data: [LLMObsPromptVersionListData] + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_prompts_response.py b/src/datadog_api_client/v2/model/llm_obs_prompts_response.py new file mode 100644 index 0000000000..aa61532a67 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_prompts_response.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_prompt_data import LLMObsPromptData + + +class LLMObsPromptsResponse(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_prompt_data import LLMObsPromptData + + return { + "data": ([LLMObsPromptData],), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: List[LLMObsPromptData], **kwargs): + """ + Response containing a list of LLM Observability prompts. + + :param data: List of LLM Observability prompts. + :type data: [LLMObsPromptData] + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_data.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_data.py new file mode 100644 index 0000000000..8a403175a9 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_data.py @@ -0,0 +1,48 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_update_prompt_data_attributes import LLMObsUpdatePromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + +class LLMObsUpdatePromptData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_update_prompt_data_attributes import LLMObsUpdatePromptDataAttributes + from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType + + return { + "attributes": (LLMObsUpdatePromptDataAttributes,), + "type": (LLMObsPromptType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__(self_, attributes: LLMObsUpdatePromptDataAttributes, type: LLMObsPromptType, **kwargs): + """ + Data object for updating an LLM Observability prompt. + + :param attributes: Attributes for updating an LLM Observability prompt. At least one of ``title`` or ``description`` must be provided; both attributes are optional individually. + :type attributes: LLMObsUpdatePromptDataAttributes + + :param type: Resource type of an LLM Observability prompt. + :type type: LLMObsPromptType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_data_attributes.py new file mode 100644 index 0000000000..84926b413e --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_data_attributes.py @@ -0,0 +1,43 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsUpdatePromptDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + return { + "description": (str,), + "title": (str,), + } + + attribute_map = { + "description": "description", + "title": "title", + } + + def __init__(self_, description: Union[str, UnsetType] = unset, title: Union[str, UnsetType] = unset, **kwargs): + """ + Attributes for updating an LLM Observability prompt. At least one of ``title`` or ``description`` must be provided; both attributes are optional individually. + + :param description: Optional new description for the prompt. + :type description: str, optional + + :param title: Optional new title for the prompt. + :type title: str, optional + """ + if description is not unset: + kwargs["description"] = description + if title is not unset: + kwargs["title"] = title + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_request.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_request.py new file mode 100644 index 0000000000..3084a60878 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_update_prompt_data import LLMObsUpdatePromptData + + +class LLMObsUpdatePromptRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_update_prompt_data import LLMObsUpdatePromptData + + return { + "data": (LLMObsUpdatePromptData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsUpdatePromptData, **kwargs): + """ + Request to update an LLM Observability prompt's metadata. + + :param data: Data object for updating an LLM Observability prompt. + :type data: LLMObsUpdatePromptData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data.py new file mode 100644 index 0000000000..622f0f53da --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data.py @@ -0,0 +1,52 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_update_prompt_version_data_attributes import ( + LLMObsUpdatePromptVersionDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + +class LLMObsUpdatePromptVersionData(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_update_prompt_version_data_attributes import ( + LLMObsUpdatePromptVersionDataAttributes, + ) + from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType + + return { + "attributes": (LLMObsUpdatePromptVersionDataAttributes,), + "type": (LLMObsPromptVersionType,), + } + + attribute_map = { + "attributes": "attributes", + "type": "type", + } + + def __init__(self_, attributes: LLMObsUpdatePromptVersionDataAttributes, type: LLMObsPromptVersionType, **kwargs): + """ + Data object for updating an LLM Observability prompt version. + + :param attributes: Attributes for updating an LLM Observability prompt version. At least one of ``description`` , ``labels`` , or ``env_ids`` must be provided; all three attributes are optional individually. + :type attributes: LLMObsUpdatePromptVersionDataAttributes + + :param type: Resource type of an LLM Observability prompt version. + :type type: LLMObsPromptVersionType + """ + super().__init__(kwargs) + + self_.attributes = attributes + self_.type = type diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data_attributes.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data_attributes.py new file mode 100644 index 0000000000..b9ccb2faed --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_data_attributes.py @@ -0,0 +1,56 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class LLMObsUpdatePromptVersionDataAttributes(ModelNormal): + @cached_property + def openapi_types(_): + return { + "description": (str,), + "env_ids": ([str],), + "labels": ([str],), + } + + attribute_map = { + "description": "description", + "env_ids": "env_ids", + "labels": "labels", + } + + def __init__( + self_, + description: Union[str, UnsetType] = unset, + env_ids: Union[List[str], UnsetType] = unset, + labels: Union[List[str], UnsetType] = unset, + **kwargs, + ): + """ + Attributes for updating an LLM Observability prompt version. At least one of ``description`` , ``labels`` , or ``env_ids`` must be provided; all three attributes are optional individually. + + :param description: Optional new description for this version. + :type description: str, optional + + :param env_ids: Optional feature-flag environment UUIDs whose default variant should be set to this version. + :type env_ids: [str], optional + + :param labels: Optional new labels for this version. Do not use this attribute for new integrations. **Deprecated**. + :type labels: [str], optional + """ + if description is not unset: + kwargs["description"] = description + if env_ids is not unset: + kwargs["env_ids"] = env_ids + if labels is not unset: + kwargs["labels"] = labels + super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_request.py b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_request.py new file mode 100644 index 0000000000..10e4e22d95 --- /dev/null +++ b/src/datadog_api_client/v2/model/llm_obs_update_prompt_version_request.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.llm_obs_update_prompt_version_data import LLMObsUpdatePromptVersionData + + +class LLMObsUpdatePromptVersionRequest(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.llm_obs_update_prompt_version_data import LLMObsUpdatePromptVersionData + + return { + "data": (LLMObsUpdatePromptVersionData,), + } + + attribute_map = { + "data": "data", + } + + def __init__(self_, data: LLMObsUpdatePromptVersionData, **kwargs): + """ + Request to update an LLM Observability prompt version's metadata or feature-flag environments. + + :param data: Data object for updating an LLM Observability prompt version. + :type data: LLMObsUpdatePromptVersionData + """ + super().__init__(kwargs) + + self_.data = data diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index 7db779d266..a635e41269 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3953,6 +3953,14 @@ ) from datadog_api_client.v2.model.llm_obs_content_block_time_frame import LLMObsContentBlockTimeFrame from datadog_api_client.v2.model.llm_obs_content_block_type import LLMObsContentBlockType +from datadog_api_client.v2.model.llm_obs_create_prompt_data import LLMObsCreatePromptData +from datadog_api_client.v2.model.llm_obs_create_prompt_data_attributes import LLMObsCreatePromptDataAttributes +from datadog_api_client.v2.model.llm_obs_create_prompt_request import LLMObsCreatePromptRequest +from datadog_api_client.v2.model.llm_obs_create_prompt_version_data import LLMObsCreatePromptVersionData +from datadog_api_client.v2.model.llm_obs_create_prompt_version_data_attributes import ( + LLMObsCreatePromptVersionDataAttributes, +) +from datadog_api_client.v2.model.llm_obs_create_prompt_version_request import LLMObsCreatePromptVersionRequest from datadog_api_client.v2.model.llm_obs_cursor_meta import LLMObsCursorMeta from datadog_api_client.v2.model.llm_obs_custom_eval_config_assessment_criteria import ( LLMObsCustomEvalConfigAssessmentCriteria, @@ -4105,6 +4113,9 @@ ) from datadog_api_client.v2.model.llm_obs_delete_projects_data_request import LLMObsDeleteProjectsDataRequest from datadog_api_client.v2.model.llm_obs_delete_projects_request import LLMObsDeleteProjectsRequest +from datadog_api_client.v2.model.llm_obs_deleted_prompt_data import LLMObsDeletedPromptData +from datadog_api_client.v2.model.llm_obs_deleted_prompt_data_attributes import LLMObsDeletedPromptDataAttributes +from datadog_api_client.v2.model.llm_obs_deleted_prompt_response import LLMObsDeletedPromptResponse from datadog_api_client.v2.model.llm_obs_display_block_annotated_interaction_item import ( LLMObsDisplayBlockAnnotatedInteractionItem, ) @@ -4336,6 +4347,27 @@ from datadog_api_client.v2.model.llm_obs_project_update_data_request import LLMObsProjectUpdateDataRequest from datadog_api_client.v2.model.llm_obs_project_update_request import LLMObsProjectUpdateRequest from datadog_api_client.v2.model.llm_obs_projects_response import LLMObsProjectsResponse +from datadog_api_client.v2.model.llm_obs_prompt_chat_message import LLMObsPromptChatMessage +from datadog_api_client.v2.model.llm_obs_prompt_data import LLMObsPromptData +from datadog_api_client.v2.model.llm_obs_prompt_data_attributes import LLMObsPromptDataAttributes +from datadog_api_client.v2.model.llm_obs_prompt_dataset import LLMObsPromptDataset +from datadog_api_client.v2.model.llm_obs_prompt_response import LLMObsPromptResponse +from datadog_api_client.v2.model.llm_obs_prompt_response_source import LLMObsPromptResponseSource +from datadog_api_client.v2.model.llm_obs_prompt_sdk_data import LLMObsPromptSDKData +from datadog_api_client.v2.model.llm_obs_prompt_sdk_data_attributes import LLMObsPromptSDKDataAttributes +from datadog_api_client.v2.model.llm_obs_prompt_sdk_response import LLMObsPromptSDKResponse +from datadog_api_client.v2.model.llm_obs_prompt_template import LLMObsPromptTemplate +from datadog_api_client.v2.model.llm_obs_prompt_type import LLMObsPromptType +from datadog_api_client.v2.model.llm_obs_prompt_version_data import LLMObsPromptVersionData +from datadog_api_client.v2.model.llm_obs_prompt_version_data_attributes import LLMObsPromptVersionDataAttributes +from datadog_api_client.v2.model.llm_obs_prompt_version_list_data import LLMObsPromptVersionListData +from datadog_api_client.v2.model.llm_obs_prompt_version_list_data_attributes import ( + LLMObsPromptVersionListDataAttributes, +) +from datadog_api_client.v2.model.llm_obs_prompt_version_response import LLMObsPromptVersionResponse +from datadog_api_client.v2.model.llm_obs_prompt_version_type import LLMObsPromptVersionType +from datadog_api_client.v2.model.llm_obs_prompt_versions_response import LLMObsPromptVersionsResponse +from datadog_api_client.v2.model.llm_obs_prompts_response import LLMObsPromptsResponse from datadog_api_client.v2.model.llm_obs_record_type import LLMObsRecordType from datadog_api_client.v2.model.llm_obs_search_spans_request import LLMObsSearchSpansRequest from datadog_api_client.v2.model.llm_obs_search_spans_request_attributes import LLMObsSearchSpansRequestAttributes @@ -4361,6 +4393,14 @@ from datadog_api_client.v2.model.llm_obs_trace_interaction_item import LLMObsTraceInteractionItem from datadog_api_client.v2.model.llm_obs_trace_interaction_response_item import LLMObsTraceInteractionResponseItem from datadog_api_client.v2.model.llm_obs_trace_interaction_type import LLMObsTraceInteractionType +from datadog_api_client.v2.model.llm_obs_update_prompt_data import LLMObsUpdatePromptData +from datadog_api_client.v2.model.llm_obs_update_prompt_data_attributes import LLMObsUpdatePromptDataAttributes +from datadog_api_client.v2.model.llm_obs_update_prompt_request import LLMObsUpdatePromptRequest +from datadog_api_client.v2.model.llm_obs_update_prompt_version_data import LLMObsUpdatePromptVersionData +from datadog_api_client.v2.model.llm_obs_update_prompt_version_data_attributes import ( + LLMObsUpdatePromptVersionDataAttributes, +) +from datadog_api_client.v2.model.llm_obs_update_prompt_version_request import LLMObsUpdatePromptVersionRequest from datadog_api_client.v2.model.llm_obs_upsert_annotation_item import LLMObsUpsertAnnotationItem from datadog_api_client.v2.model.llm_obs_vertex_ai_metadata import LLMObsVertexAIMetadata from datadog_api_client.v2.model.language import Language @@ -12899,6 +12939,12 @@ "LLMObsContentBlockLLMObsTraceInteractionType", "LLMObsContentBlockTimeFrame", "LLMObsContentBlockType", + "LLMObsCreatePromptData", + "LLMObsCreatePromptDataAttributes", + "LLMObsCreatePromptRequest", + "LLMObsCreatePromptVersionData", + "LLMObsCreatePromptVersionDataAttributes", + "LLMObsCreatePromptVersionRequest", "LLMObsCursorMeta", "LLMObsCustomEvalConfigAssessmentCriteria", "LLMObsCustomEvalConfigAttributes", @@ -12999,6 +13045,9 @@ "LLMObsDeleteProjectsDataAttributesRequest", "LLMObsDeleteProjectsDataRequest", "LLMObsDeleteProjectsRequest", + "LLMObsDeletedPromptData", + "LLMObsDeletedPromptDataAttributes", + "LLMObsDeletedPromptResponse", "LLMObsDisplayBlockAnnotatedInteractionItem", "LLMObsDisplayBlockInteractionItem", "LLMObsDisplayBlockInteractionResponseItem", @@ -13152,6 +13201,25 @@ "LLMObsProjectUpdateDataRequest", "LLMObsProjectUpdateRequest", "LLMObsProjectsResponse", + "LLMObsPromptChatMessage", + "LLMObsPromptData", + "LLMObsPromptDataAttributes", + "LLMObsPromptDataset", + "LLMObsPromptResponse", + "LLMObsPromptResponseSource", + "LLMObsPromptSDKData", + "LLMObsPromptSDKDataAttributes", + "LLMObsPromptSDKResponse", + "LLMObsPromptTemplate", + "LLMObsPromptType", + "LLMObsPromptVersionData", + "LLMObsPromptVersionDataAttributes", + "LLMObsPromptVersionListData", + "LLMObsPromptVersionListDataAttributes", + "LLMObsPromptVersionResponse", + "LLMObsPromptVersionType", + "LLMObsPromptVersionsResponse", + "LLMObsPromptsResponse", "LLMObsRecordType", "LLMObsSearchSpansRequest", "LLMObsSearchSpansRequestAttributes", @@ -13177,6 +13245,12 @@ "LLMObsTraceInteractionItem", "LLMObsTraceInteractionResponseItem", "LLMObsTraceInteractionType", + "LLMObsUpdatePromptData", + "LLMObsUpdatePromptDataAttributes", + "LLMObsUpdatePromptRequest", + "LLMObsUpdatePromptVersionData", + "LLMObsUpdatePromptVersionDataAttributes", + "LLMObsUpdatePromptVersionRequest", "LLMObsUpsertAnnotationItem", "LLMObsVertexAIMetadata", "Language", diff --git a/tests/v2/features/given.json b/tests/v2/features/given.json index 0f9de6d48a..d0cfa23259 100644 --- a/tests/v2/features/given.json +++ b/tests/v2/features/given.json @@ -858,6 +858,34 @@ "tag": "IP Allowlist", "operationId": "UpdateIPAllowlist" }, + { + "parameters": [ + { + "name": "body", + "value": "{\n \"data\": {\n \"type\": \"prompt-templates\",\n \"attributes\": {\n \"prompt_id\": \"{{ unique }}\",\n \"template\": [{\"role\": \"user\", \"content\": \"Hello\"}]\n }\n }\n}" + } + ], + "step": "there is a valid \"prompt\" in the system", + "key": "prompt", + "tag": "LLM Observability", + "operationId": "CreateLLMObsPrompt" + }, + { + "parameters": [ + { + "name": "prompt_id", + "source": "prompt.data.attributes.prompt_id" + }, + { + "name": "body", + "value": "{\n \"data\": {\n \"type\": \"prompt-template-versions\",\n \"attributes\": {\n \"template\": [{\"role\": \"user\", \"content\": \"Hello v2\"}]\n }\n }\n}" + } + ], + "step": "there is a valid \"prompt_version\" in the system", + "key": "prompt_version", + "tag": "LLM Observability", + "operationId": "CreateLLMObsPromptVersion" + }, { "parameters": [ { diff --git a/tests/v2/features/llm_observability.feature b/tests/v2/features/llm_observability.feature index 35e9592fa0..13a928fc8d 100644 --- a/tests/v2/features/llm_observability.feature +++ b/tests/v2/features/llm_observability.feature @@ -1,16 +1,16 @@ @endpoint(llm-observability) @endpoint(llm-observability-v2) Feature: LLM Observability Manage LLM Observability spans, data, projects, datasets, dataset records, - experiments, and annotations. + experiments, prompts, and annotations. Background: Given a valid "apiKeyAuth" key in the system - And a valid "appKeyAuth" key in the system And an instance of "LLMObservability" API @skip @team:DataDog/ml-observability Scenario: Add a display_block interaction returns "Created" response - Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueueInteractions" enabled And new "CreateLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interactions": [{"type": "display_block", "display_block": [{"type": "markdown", "content": "## Triage Instructions"}]}]}, "type": "interactions"}} @@ -19,7 +19,8 @@ Feature: LLM Observability @skip @team:DataDog/ml-observability Scenario: Add a display_block interaction with an image block missing url returns "Bad Request" response - Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueueInteractions" enabled And new "CreateLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interactions": [{"type": "display_block", "display_block": [{"type": "image"}]}]}, "type": "interactions"}} @@ -28,7 +29,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Add annotation queue interactions returns "Bad Request" response - Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueueInteractions" enabled And new "CreateLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} @@ -37,7 +39,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Add annotation queue interactions returns "Created" response - Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueueInteractions" enabled And new "CreateLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} @@ -46,7 +49,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Add annotation queue interactions returns "Not Found" response - Given operation "CreateLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueueInteractions" enabled And new "CreateLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interactions": [{"content_id": "trace-abc-123", "type": "trace"}]}, "type": "interactions"}} @@ -55,7 +59,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Aggregate LLM Observability experimentation returns "Bad Request" response - Given operation "AggregateLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "AggregateLLMObsExperimentation" enabled And new "AggregateLLMObsExperimentation" request And body with value {"data": {"attributes": {"aggregate": {"compute": [{"metric": "score_value", "name": "avg_faithfulness"}], "dataset_version": null, "group_by": [{"field": "span_id"}], "indexes": ["experiment-evals"], "limit": 1000, "search": {"query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"}, "time": {"from": 1705312200000, "to": 1705315800000}}}, "type": "experimentation"}} When the request is sent @@ -63,7 +68,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Aggregate LLM Observability experimentation returns "OK" response - Given operation "AggregateLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "AggregateLLMObsExperimentation" enabled And new "AggregateLLMObsExperimentation" request And body with value {"data": {"attributes": {"aggregate": {"compute": [{"metric": "score_value", "name": "avg_faithfulness"}], "dataset_version": null, "group_by": [{"field": "span_id"}], "indexes": ["experiment-evals"], "limit": 1000, "search": {"query": "@experiment_id:3fd6b5e0-8910-4b1c-a7d0-5b84de329012"}, "time": {"from": 1705312200000, "to": 1705315800000}}}, "type": "experimentation"}} When the request is sent @@ -71,7 +77,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Append records to an LLM Observability dataset returns "Bad Request" response - Given operation "CreateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDatasetRecords" enabled And new "CreateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -81,7 +88,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Append records to an LLM Observability dataset returns "Created" response - Given operation "CreateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDatasetRecords" enabled And new "CreateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -91,7 +99,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Append records to an LLM Observability dataset returns "Not Found" response - Given operation "CreateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDatasetRecords" enabled And new "CreateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -101,7 +110,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Append records to an LLM Observability dataset returns "OK" response - Given operation "CreateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDatasetRecords" enabled And new "CreateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -111,7 +121,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Batch update LLM Observability dataset records returns "Bad Request" response - Given operation "BatchUpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "BatchUpdateLLMObsDataset" enabled And new "BatchUpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -121,7 +132,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Batch update LLM Observability dataset records returns "Not Found" response - Given operation "BatchUpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "BatchUpdateLLMObsDataset" enabled And new "BatchUpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -131,7 +143,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Batch update LLM Observability dataset records returns "OK" response - Given operation "BatchUpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "BatchUpdateLLMObsDataset" enabled And new "BatchUpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -141,7 +154,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Batch update LLM Observability dataset records returns "Payload Too Large" response - Given operation "BatchUpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "BatchUpdateLLMObsDataset" enabled And new "BatchUpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -151,7 +165,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Clone an LLM Observability dataset returns "Bad Request" response - Given operation "CloneLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CloneLLMObsDataset" enabled And new "CloneLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -161,7 +176,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Clone an LLM Observability dataset returns "Not Found" response - Given operation "CloneLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CloneLLMObsDataset" enabled And new "CloneLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -171,7 +187,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Clone an LLM Observability dataset returns "OK" response - Given operation "CloneLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CloneLLMObsDataset" enabled And new "CloneLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -179,9 +196,54 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: Create a new LLM Observability prompt version returns "Bad Request" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPromptVersion" enabled + And new "CreateLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And body with value {} + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: Create a new LLM Observability prompt version returns "Conflict" response + Given there is a valid "prompt" in the system + And there is a valid "prompt_version" in the system + And a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPromptVersion" enabled + And new "CreateLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And body with value {"data": {"attributes": {"env_ids": [], "labels": [], "template": [{"content": "Hello v2", "role": "user"}]}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 409 Conflict + + @skip @team:DataDog/ml-observability + Scenario: Create a new LLM Observability prompt version returns "Not Found" response + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPromptVersion" enabled + And new "CreateLLMObsPromptVersion" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + And body with value {"data": {"attributes": {"env_ids": [], "labels": [], "template": [{"content": "Hello v2", "role": "user"}]}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Create a new LLM Observability prompt version returns "OK" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPromptVersion" enabled + And new "CreateLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And body with value {"data": {"attributes": {"env_ids": [], "labels": [], "template": "Hello v3"}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability annotation queue returns "Bad Request" response - Given operation "CreateLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueue" enabled And new "CreateLLMObsAnnotationQueue" request And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "00000000-0000-0000-0000-000000000002"}, "type": "queues"}} When the request is sent @@ -189,7 +251,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability annotation queue returns "Created" response - Given operation "CreateLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsAnnotationQueue" enabled And new "CreateLLMObsAnnotationQueue" request And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Queue for annotating customer support traces", "name": "My annotation queue", "project_id": "00000000-0000-0000-0000-000000000002"}, "type": "queues"}} When the request is sent @@ -197,7 +260,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability dataset returns "Bad Request" response - Given operation "CreateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDataset" enabled And new "CreateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"name": "My LLM Dataset"}, "type": "datasets"}} @@ -206,7 +270,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability dataset returns "Created" response - Given operation "CreateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDataset" enabled And new "CreateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"name": "My LLM Dataset"}, "type": "datasets"}} @@ -215,7 +280,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability dataset returns "Not Found" response - Given operation "CreateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDataset" enabled And new "CreateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"name": "My LLM Dataset"}, "type": "datasets"}} @@ -224,7 +290,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability dataset returns "OK" response - Given operation "CreateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsDataset" enabled And new "CreateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"name": "My LLM Dataset"}, "type": "datasets"}} @@ -233,7 +300,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability experiment returns "Bad Request" response - Given operation "CreateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent @@ -241,7 +309,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability experiment returns "Created" response - Given operation "CreateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent @@ -249,7 +318,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability experiment returns "OK" response - Given operation "CreateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperiment" enabled And new "CreateLLMObsExperiment" request And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "name": "My Experiment v1", "parent_experiment_id": "3fd6b5e0-8910-4b1c-a7d0-5b84de329012", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751"}, "type": "experiments"}} When the request is sent @@ -257,7 +327,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability project returns "Bad Request" response - Given operation "CreateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsProject" enabled And new "CreateLLMObsProject" request And body with value {"data": {"attributes": {"name": "My LLM Project"}, "type": "projects"}} When the request is sent @@ -265,7 +336,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability project returns "Created" response - Given operation "CreateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsProject" enabled And new "CreateLLMObsProject" request And body with value {"data": {"attributes": {"name": "My LLM Project"}, "type": "projects"}} When the request is sent @@ -273,15 +345,45 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create an LLM Observability project returns "OK" response - Given operation "CreateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsProject" enabled And new "CreateLLMObsProject" request And body with value {"data": {"attributes": {"name": "My LLM Project"}, "type": "projects"}} When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: Create an LLM Observability prompt returns "Bad Request" response + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPrompt" enabled + And new "CreateLLMObsPrompt" request + And body with value {} + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: Create an LLM Observability prompt returns "Conflict" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPrompt" enabled + And new "CreateLLMObsPrompt" request + And body with value {"data": {"attributes": {"env_ids": [], "labels": [], "prompt_id": "{{ prompt.data.attributes.prompt_id }}", "template": [{"content": "Hello", "role": "user"}]}, "type": "prompt-templates"}} + When the request is sent + Then the response status is 409 Conflict + + @skip @team:DataDog/ml-observability + Scenario: Create an LLM Observability prompt returns "OK" response + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsPrompt" enabled + And new "CreateLLMObsPrompt" request + And body with value {"data": {"attributes": {"env_ids": [], "labels": [], "prompt_id": "{{ unique }}", "template": "Hello"}, "type": "prompt-templates"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Create or update a custom evaluator configuration returns "Bad Request" response - Given operation "UpdateLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} @@ -290,7 +392,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a custom evaluator configuration returns "Not Found" response - Given operation "UpdateLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} @@ -299,7 +402,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a custom evaluator configuration returns "OK" response - Given operation "UpdateLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} @@ -308,7 +412,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a custom evaluator configuration returns "Unprocessable Entity" response - Given operation "UpdateLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsCustomEvalConfig" enabled And new "UpdateLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"category": "Custom", "eval_name": "my-custom-evaluator", "llm_judge_config": {"assessment_criteria": {"max_threshold": 1.0, "min_threshold": 0.7, "pass_values": ["pass", "yes"], "pass_when": true}, "inference_params": {"frequency_penalty": 0.0, "max_tokens": 1024, "presence_penalty": 0.0, "temperature": 0.7, "top_k": 50, "top_p": 1.0}, "last_used_library_prompt_template_name": "sentiment-analysis-v1", "modified_library_prompt_template": false, "output_schema": null, "parsing_type": "structured_output", "prompt_template": [{"content": "Rate the quality of the following response:", "contents": [{"type": "text", "value": {"text": "What is the sentiment of this review?", "tool_call": {"arguments": "{\"location\": \"San Francisco\"}", "id": "call_abc123", "name": "get_weather", "type": "function"}, "tool_call_result": {"name": "get_weather", "result": "sunny, 72F", "tool_id": "call_abc123", "type": "function"}}}], "role": "user"}]}, "llm_provider": {"bedrock": {"region": "us-east-1"}, "integration_account_id": "my-account-id", "integration_provider": "openai", "model_name": "gpt-4o", "vertex_ai": {"location": "us-central1", "project": "my-gcp-project"}}, "target": {"application_name": "my-llm-app", "enabled": true, "eval_scope": "span", "filter": "@service:my-service", "root_spans_only": true, "sampling_percentage": 50.0}}, "id": "my-custom-evaluator", "type": "evaluator_config"}} @@ -317,7 +422,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a patterns configuration returns "Bad Request" response - Given operation "UpsertLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsPatternsConfig" enabled And new "UpsertLLMObsPatternsConfig" request And body with value {"data": {"attributes": {"account_id": "1000000001", "config_id": "a7c8d9e0-1234-5678-9abc-def012345678", "evp_query": "@ml_app:support-bot", "hierarchy_depth": 2, "integration_provider": "openai", "model_name": "gpt-4o", "name": "Support chatbot topics", "num_records": 1000, "sampling_ratio": 0.1, "scope": "", "template": ""}, "type": "topic_discovery_configs"}} When the request is sent @@ -325,7 +431,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a patterns configuration returns "Not Found" response - Given operation "UpsertLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsPatternsConfig" enabled And new "UpsertLLMObsPatternsConfig" request And body with value {"data": {"attributes": {"account_id": "1000000001", "config_id": "a7c8d9e0-1234-5678-9abc-def012345678", "evp_query": "@ml_app:support-bot", "hierarchy_depth": 2, "integration_provider": "openai", "model_name": "gpt-4o", "name": "Support chatbot topics", "num_records": 1000, "sampling_ratio": 0.1, "scope": "", "template": ""}, "type": "topic_discovery_configs"}} When the request is sent @@ -333,7 +440,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update a patterns configuration returns "OK" response - Given operation "UpsertLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsPatternsConfig" enabled And new "UpsertLLMObsPatternsConfig" request And body with value {"data": {"attributes": {"account_id": "1000000001", "config_id": "a7c8d9e0-1234-5678-9abc-def012345678", "evp_query": "@ml_app:support-bot", "hierarchy_depth": 2, "integration_provider": "openai", "model_name": "gpt-4o", "name": "Support chatbot topics", "num_records": 1000, "sampling_ratio": 0.1, "scope": "", "template": ""}, "type": "topic_discovery_configs"}} When the request is sent @@ -341,7 +449,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update annotations returns "Bad Request" response - Given operation "UpsertLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsAnnotations" enabled And new "UpsertLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotations": [{"interaction_id": "00000000-0000-0000-0000-000000000001", "label_values": [{"label_schema_id": "abc-123", "value": "good"}, {"label_schema_id": "ef56gh78", "value": "positive"}]}]}, "type": "annotations"}} @@ -350,7 +459,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update annotations returns "Not Found — the queue does not exist." response - Given operation "UpsertLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsAnnotations" enabled And new "UpsertLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotations": [{"interaction_id": "00000000-0000-0000-0000-000000000001", "label_values": [{"label_schema_id": "abc-123", "value": "good"}, {"label_schema_id": "ef56gh78", "value": "positive"}]}]}, "type": "annotations"}} @@ -359,7 +469,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Create or update annotations returns "OK — annotations created or updated. Per-item errors are listed in `errors`." response - Given operation "UpsertLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpsertLLMObsAnnotations" enabled And new "UpsertLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotations": [{"interaction_id": "00000000-0000-0000-0000-000000000001", "label_values": [{"label_schema_id": "abc-123", "value": "good"}, {"label_schema_id": "ef56gh78", "value": "positive"}]}]}, "type": "annotations"}} @@ -368,7 +479,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability data returns "Accepted" response - Given operation "DeleteLLMObsData" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsData" enabled And new "DeleteLLMObsData" request And body with value {"data": {"attributes": {"delay": 0, "from": 1705314600000, "query": {"query": "@trace_id:abc123def456"}, "to": 1705315200000}, "type": "create_deletion_req"}} When the request is sent @@ -376,7 +488,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability data returns "Bad Request" response - Given operation "DeleteLLMObsData" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsData" enabled And new "DeleteLLMObsData" request And body with value {"data": {"attributes": {"delay": 0, "from": 1705314600000, "query": {"query": "@trace_id:abc123def456"}, "to": 1705315200000}, "type": "create_deletion_req"}} When the request is sent @@ -384,7 +497,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability dataset records returns "Bad Request" response - Given operation "DeleteLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasetRecords" enabled And new "DeleteLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -394,7 +508,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability dataset records returns "No Content" response - Given operation "DeleteLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasetRecords" enabled And new "DeleteLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -404,7 +519,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability dataset records returns "Not Found" response - Given operation "DeleteLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasetRecords" enabled And new "DeleteLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -414,7 +530,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability datasets returns "Bad Request" response - Given operation "DeleteLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasets" enabled And new "DeleteLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_ids": ["9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d"]}, "type": "datasets"}} @@ -423,7 +540,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability datasets returns "No Content" response - Given operation "DeleteLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasets" enabled And new "DeleteLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_ids": ["9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d"]}, "type": "datasets"}} @@ -432,7 +550,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability datasets returns "Not Found" response - Given operation "DeleteLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsDatasets" enabled And new "DeleteLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_ids": ["9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d"]}, "type": "datasets"}} @@ -441,7 +560,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability experiments returns "Bad Request" response - Given operation "DeleteLLMObsExperiments" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsExperiments" enabled And new "DeleteLLMObsExperiments" request And body with value {"data": {"attributes": {"experiment_ids": ["3fd6b5e0-8910-4b1c-a7d0-5b84de329012"]}, "type": "experiments"}} When the request is sent @@ -449,7 +569,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability experiments returns "No Content" response - Given operation "DeleteLLMObsExperiments" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsExperiments" enabled And new "DeleteLLMObsExperiments" request And body with value {"data": {"attributes": {"experiment_ids": ["3fd6b5e0-8910-4b1c-a7d0-5b84de329012"]}, "type": "experiments"}} When the request is sent @@ -457,7 +578,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability projects returns "Bad Request" response - Given operation "DeleteLLMObsProjects" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsProjects" enabled And new "DeleteLLMObsProjects" request And body with value {"data": {"attributes": {"project_ids": ["a33671aa-24fd-4dcd-9b33-a8ec7dde7751"]}, "type": "projects"}} When the request is sent @@ -465,7 +587,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete LLM Observability projects returns "No Content" response - Given operation "DeleteLLMObsProjects" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsProjects" enabled And new "DeleteLLMObsProjects" request And body with value {"data": {"attributes": {"project_ids": ["a33671aa-24fd-4dcd-9b33-a8ec7dde7751"]}, "type": "projects"}} When the request is sent @@ -473,7 +596,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a custom evaluator configuration returns "Bad Request" response - Given operation "DeleteLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsCustomEvalConfig" enabled And new "DeleteLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -481,7 +605,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a custom evaluator configuration returns "No Content" response - Given operation "DeleteLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsCustomEvalConfig" enabled And new "DeleteLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -489,7 +614,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a custom evaluator configuration returns "Not Found" response - Given operation "DeleteLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsCustomEvalConfig" enabled And new "DeleteLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -497,7 +623,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a patterns configuration returns "Bad Request" response - Given operation "DeleteLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsPatternsConfig" enabled And new "DeleteLLMObsPatternsConfig" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -505,7 +632,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a patterns configuration returns "No Content" response - Given operation "DeleteLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsPatternsConfig" enabled And new "DeleteLLMObsPatternsConfig" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -513,7 +641,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete a patterns configuration returns "Not Found" response - Given operation "DeleteLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsPatternsConfig" enabled And new "DeleteLLMObsPatternsConfig" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -521,7 +650,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete an LLM Observability annotation queue returns "No Content" response - Given operation "DeleteLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotationQueue" enabled And new "DeleteLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -529,15 +659,36 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete an LLM Observability annotation queue returns "Not Found" response - Given operation "DeleteLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotationQueue" enabled And new "DeleteLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 404 Not Found + @skip @team:DataDog/ml-observability + Scenario: Delete an LLM Observability prompt returns "Not Found" response + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsPrompt" enabled + And new "DeleteLLMObsPrompt" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Delete an LLM Observability prompt returns "OK" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsPrompt" enabled + And new "DeleteLLMObsPrompt" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Delete annotation queue interactions returns "Bad Request" response - Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotationQueueInteractions" enabled And new "DeleteLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} @@ -546,7 +697,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete annotation queue interactions returns "No Content" response - Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotationQueueInteractions" enabled And new "DeleteLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} @@ -555,7 +707,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete annotation queue interactions returns "Not Found" response - Given operation "DeleteLLMObsAnnotationQueueInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotationQueueInteractions" enabled And new "DeleteLLMObsAnnotationQueueInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"interaction_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "interactions"}} @@ -564,7 +717,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete annotations returns "Bad Request" response - Given operation "DeleteLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotations" enabled And new "DeleteLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "annotations"}} @@ -573,7 +727,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete annotations returns "Not Found — the queue does not exist." response - Given operation "DeleteLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotations" enabled And new "DeleteLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "annotations"}} @@ -582,7 +737,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Delete annotations returns "OK — annotations deleted. Errors for annotations that could not be deleted are listed in `errors`." response - Given operation "DeleteLLMObsAnnotations" enabled + Given a valid "appKeyAuth" key in the system + And operation "DeleteLLMObsAnnotations" enabled And new "DeleteLLMObsAnnotations" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_ids": ["00000000-0000-0000-0000-000000000000", "00000000-0000-0000-0000-000000000001"]}, "type": "annotations"}} @@ -591,7 +747,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Export an LLM Observability dataset returns "Bad Request" response - Given operation "ExportLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "ExportLLMObsDataset" enabled And new "ExportLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -600,7 +757,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Export an LLM Observability dataset returns "Not Found" response - Given operation "ExportLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "ExportLLMObsDataset" enabled And new "ExportLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -609,7 +767,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Export an LLM Observability dataset returns "OK" response - Given operation "ExportLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "ExportLLMObsDataset" enabled And new "ExportLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -618,7 +777,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get LLM Observability dataset draft state returns "Bad Request" response - Given operation "GetLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsDatasetDraftState" enabled And new "GetLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -627,7 +787,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get LLM Observability dataset draft state returns "Not Found" response - Given operation "GetLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsDatasetDraftState" enabled And new "GetLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -636,7 +797,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get LLM Observability dataset draft state returns "OK" response - Given operation "GetLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsDatasetDraftState" enabled And new "GetLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -645,7 +807,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get a custom evaluator configuration returns "Bad Request" response - Given operation "GetLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsCustomEvalConfig" enabled And new "GetLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -653,7 +816,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get a custom evaluator configuration returns "Not Found" response - Given operation "GetLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsCustomEvalConfig" enabled And new "GetLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -661,7 +825,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get a custom evaluator configuration returns "OK" response - Given operation "GetLLMObsCustomEvalConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsCustomEvalConfig" enabled And new "GetLLMObsCustomEvalConfig" request And request contains "eval_name" parameter from "REPLACE.ME" When the request is sent @@ -669,28 +834,80 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get a patterns configuration returns "Bad Request" response - Given operation "GetLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsConfig" enabled And new "GetLLMObsPatternsConfig" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: Get a patterns configuration returns "Not Found" response - Given operation "GetLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsConfig" enabled And new "GetLLMObsPatternsConfig" request When the request is sent Then the response status is 404 Not Found @generated @skip @team:DataDog/ml-observability Scenario: Get a patterns configuration returns "OK" response - Given operation "GetLLMObsPatternsConfig" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsConfig" enabled And new "GetLLMObsPatternsConfig" request When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: Get a specific LLM Observability prompt version returns "Bad Request" response + Given there is a valid "prompt" in the system + And there is a valid "prompt_version" in the system + And operation "GetLLMObsPromptVersion" enabled + And new "GetLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And request contains "version" parameter with value "not-a-number" + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: Get a specific LLM Observability prompt version returns "Not Found" response + Given operation "GetLLMObsPromptVersion" enabled + And new "GetLLMObsPromptVersion" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + And request contains "version" parameter with value "1" + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Get a specific LLM Observability prompt version returns "OK" response + Given there is a valid "prompt" in the system + And there is a valid "prompt_version" in the system + And operation "GetLLMObsPromptVersion" enabled + And new "GetLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And request contains "version" parameter from "prompt_version.data.attributes.version" + When the request is sent + Then the response status is 200 OK + + @skip @team:DataDog/ml-observability + Scenario: Get an LLM Observability prompt returns "Not Found" response + Given operation "GetLLMObsPrompt" enabled + And new "GetLLMObsPrompt" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Get an LLM Observability prompt returns "OK" response + Given there is a valid "prompt" in the system + And operation "GetLLMObsPrompt" enabled + And new "GetLLMObsPrompt" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Get annotated interactions by content IDs returns "Bad Request" response - Given operation "GetLLMObsAnnotatedInteractionsByTraceIDs" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotatedInteractionsByTraceIDs" enabled And new "GetLLMObsAnnotatedInteractionsByTraceIDs" request And request contains "contentIds" parameter from "REPLACE.ME" When the request is sent @@ -698,7 +915,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotated interactions by content IDs returns "OK" response - Given operation "GetLLMObsAnnotatedInteractionsByTraceIDs" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotatedInteractionsByTraceIDs" enabled And new "GetLLMObsAnnotatedInteractionsByTraceIDs" request And request contains "contentIds" parameter from "REPLACE.ME" When the request is sent @@ -706,7 +924,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotated queue interactions returns "Bad Request" response - Given operation "GetLLMObsAnnotatedInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotatedInteractions" enabled And new "GetLLMObsAnnotatedInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -714,7 +933,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotated queue interactions returns "Not Found" response - Given operation "GetLLMObsAnnotatedInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotatedInteractions" enabled And new "GetLLMObsAnnotatedInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -722,7 +942,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotated queue interactions returns "OK" response - Given operation "GetLLMObsAnnotatedInteractions" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotatedInteractions" enabled And new "GetLLMObsAnnotatedInteractions" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -730,7 +951,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotation queue label schema returns "Not Found" response - Given operation "GetLLMObsAnnotationQueueLabelSchema" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotationQueueLabelSchema" enabled And new "GetLLMObsAnnotationQueueLabelSchema" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -738,7 +960,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get annotation queue label schema returns "OK" response - Given operation "GetLLMObsAnnotationQueueLabelSchema" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsAnnotationQueueLabelSchema" enabled And new "GetLLMObsAnnotationQueueLabelSchema" request And request contains "queue_id" parameter from "REPLACE.ME" When the request is sent @@ -746,7 +969,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get patterns run status returns "Bad Request" response - Given operation "GetLLMObsPatternsRunStatus" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsRunStatus" enabled And new "GetLLMObsPatternsRunStatus" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -754,7 +978,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get patterns run status returns "Not Found" response - Given operation "GetLLMObsPatternsRunStatus" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsRunStatus" enabled And new "GetLLMObsPatternsRunStatus" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -762,7 +987,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Get patterns run status returns "OK" response - Given operation "GetLLMObsPatternsRunStatus" enabled + Given a valid "appKeyAuth" key in the system + And operation "GetLLMObsPatternsRunStatus" enabled And new "GetLLMObsPatternsRunStatus" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -770,21 +996,24 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability annotation queues returns "Bad Request" response - Given operation "ListLLMObsAnnotationQueues" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsAnnotationQueues" enabled And new "ListLLMObsAnnotationQueues" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability annotation queues returns "OK" response - Given operation "ListLLMObsAnnotationQueues" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsAnnotationQueues" enabled And new "ListLLMObsAnnotationQueues" request When the request is sent Then the response status is 200 OK @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset records returns "Bad Request" response - Given operation "ListLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetRecords" enabled And new "ListLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -793,7 +1022,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset records returns "Not Found" response - Given operation "ListLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetRecords" enabled And new "ListLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -802,7 +1032,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset records returns "OK" response - Given operation "ListLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetRecords" enabled And new "ListLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -811,7 +1042,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset versions returns "Bad Request" response - Given operation "ListLLMObsDatasetVersions" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetVersions" enabled And new "ListLLMObsDatasetVersions" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -820,7 +1052,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset versions returns "Not Found" response - Given operation "ListLLMObsDatasetVersions" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetVersions" enabled And new "ListLLMObsDatasetVersions" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -829,7 +1062,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability dataset versions returns "OK" response - Given operation "ListLLMObsDatasetVersions" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasetVersions" enabled And new "ListLLMObsDatasetVersions" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -838,7 +1072,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability datasets returns "Bad Request" response - Given operation "ListLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasets" enabled And new "ListLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" When the request is sent @@ -846,7 +1081,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability datasets returns "Not Found" response - Given operation "ListLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasets" enabled And new "ListLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" When the request is sent @@ -854,7 +1090,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability datasets returns "OK" response - Given operation "ListLLMObsDatasets" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsDatasets" enabled And new "ListLLMObsDatasets" request And request contains "project_id" parameter from "REPLACE.ME" When the request is sent @@ -862,7 +1099,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment events (v2) returns "Bad Request" response - Given operation "ListLLMObsExperimentEventsV2" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV2" enabled And new "ListLLMObsExperimentEventsV2" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -870,7 +1108,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment events (v2) returns "Not Found" response - Given operation "ListLLMObsExperimentEventsV2" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV2" enabled And new "ListLLMObsExperimentEventsV2" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -878,7 +1117,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment events (v2) returns "OK" response - Given operation "ListLLMObsExperimentEventsV2" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV2" enabled And new "ListLLMObsExperimentEventsV2" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -886,7 +1126,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment spans (v1) returns "Bad Request" response - Given operation "ListLLMObsExperimentEventsV1" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV1" enabled And new "ListLLMObsExperimentEventsV1" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -894,7 +1135,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment spans (v1) returns "Not Found" response - Given operation "ListLLMObsExperimentEventsV1" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV1" enabled And new "ListLLMObsExperimentEventsV1" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -902,7 +1144,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiment spans (v1) returns "OK" response - Given operation "ListLLMObsExperimentEventsV1" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEventsV1" enabled And new "ListLLMObsExperimentEventsV1" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -910,49 +1153,72 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiments returns "Bad Request" response - Given operation "ListLLMObsExperiments" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperiments" enabled And new "ListLLMObsExperiments" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability experiments returns "OK" response - Given operation "ListLLMObsExperiments" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperiments" enabled And new "ListLLMObsExperiments" request When the request is sent Then the response status is 200 OK @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability projects returns "Bad Request" response - Given operation "ListLLMObsProjects" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsProjects" enabled And new "ListLLMObsProjects" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability projects returns "OK" response - Given operation "ListLLMObsProjects" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsProjects" enabled And new "ListLLMObsProjects" request When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: List LLM Observability prompts returns "Bad Request" response + Given operation "ListLLMObsPrompts" enabled + And new "ListLLMObsPrompts" request + And request contains "filter[ml_app]" parameter with value "" + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: List LLM Observability prompts returns "OK" response + Given there is a valid "prompt" in the system + And operation "ListLLMObsPrompts" enabled + And new "ListLLMObsPrompts" request + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability spans returns "Bad Request" response - Given operation "ListLLMObsSpans" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsSpans" enabled And new "ListLLMObsSpans" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: List LLM Observability spans returns "OK" response - Given operation "ListLLMObsSpans" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsSpans" enabled And new "ListLLMObsSpans" request When the request is sent Then the response status is 200 OK @generated @skip @team:DataDog/ml-observability Scenario: List LLM integration accounts returns "Bad Request" response - Given operation "ListLLMObsIntegrationAccounts" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsIntegrationAccounts" enabled And new "ListLLMObsIntegrationAccounts" request And request contains "integration" parameter from "REPLACE.ME" When the request is sent @@ -960,7 +1226,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM integration accounts returns "OK" response - Given operation "ListLLMObsIntegrationAccounts" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsIntegrationAccounts" enabled And new "ListLLMObsIntegrationAccounts" request And request contains "integration" parameter from "REPLACE.ME" When the request is sent @@ -968,7 +1235,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM integration models returns "Bad Request" response - Given operation "ListLLMObsIntegrationModels" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsIntegrationModels" enabled And new "ListLLMObsIntegrationModels" request And request contains "integration" parameter from "REPLACE.ME" And request contains "account_id" parameter from "REPLACE.ME" @@ -977,7 +1245,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List LLM integration models returns "OK" response - Given operation "ListLLMObsIntegrationModels" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsIntegrationModels" enabled And new "ListLLMObsIntegrationModels" request And request contains "integration" parameter from "REPLACE.ME" And request contains "account_id" parameter from "REPLACE.ME" @@ -986,7 +1255,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Bad Request" response - Given operation "ListLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEvents" enabled And new "ListLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -994,7 +1264,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "Not Found" response - Given operation "ListLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEvents" enabled And new "ListLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -1002,7 +1273,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List events for an LLM Observability experiment returns "OK" response - Given operation "ListLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsExperimentEvents" enabled And new "ListLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" When the request is sent @@ -1010,7 +1282,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns clustered points returns "Bad Request" response - Given operation "ListLLMObsPatternsClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsClusteredPoints" enabled And new "ListLLMObsPatternsClusteredPoints" request And request contains "topic_id" parameter from "REPLACE.ME" When the request is sent @@ -1018,7 +1291,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns clustered points returns "Not Found" response - Given operation "ListLLMObsPatternsClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsClusteredPoints" enabled And new "ListLLMObsPatternsClusteredPoints" request And request contains "topic_id" parameter from "REPLACE.ME" When the request is sent @@ -1026,7 +1300,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns clustered points returns "OK" response - Given operation "ListLLMObsPatternsClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsClusteredPoints" enabled And new "ListLLMObsPatternsClusteredPoints" request And request contains "topic_id" parameter from "REPLACE.ME" When the request is sent @@ -1034,21 +1309,24 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns configurations returns "Bad Request" response - Given operation "ListLLMObsPatternsConfigs" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsConfigs" enabled And new "ListLLMObsPatternsConfigs" request When the request is sent Then the response status is 400 Bad Request @generated @skip @team:DataDog/ml-observability Scenario: List patterns configurations returns "OK" response - Given operation "ListLLMObsPatternsConfigs" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsConfigs" enabled And new "ListLLMObsPatternsConfigs" request When the request is sent Then the response status is 200 OK @generated @skip @team:DataDog/ml-observability Scenario: List patterns runs returns "Bad Request" response - Given operation "ListLLMObsPatternsRuns" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsRuns" enabled And new "ListLLMObsPatternsRuns" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1056,7 +1334,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns runs returns "Not Found" response - Given operation "ListLLMObsPatternsRuns" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsRuns" enabled And new "ListLLMObsPatternsRuns" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1064,7 +1343,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns runs returns "OK" response - Given operation "ListLLMObsPatternsRuns" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsRuns" enabled And new "ListLLMObsPatternsRuns" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1072,7 +1352,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics returns "Bad Request" response - Given operation "ListLLMObsPatternsTopics" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopics" enabled And new "ListLLMObsPatternsTopics" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1080,7 +1361,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics returns "Not Found" response - Given operation "ListLLMObsPatternsTopics" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopics" enabled And new "ListLLMObsPatternsTopics" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1088,7 +1370,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics returns "OK" response - Given operation "ListLLMObsPatternsTopics" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopics" enabled And new "ListLLMObsPatternsTopics" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1096,7 +1379,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics with clustered points returns "Bad Request" response - Given operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled And new "ListLLMObsPatternsTopicsWithClusteredPoints" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1104,7 +1388,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics with clustered points returns "Not Found" response - Given operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled And new "ListLLMObsPatternsTopicsWithClusteredPoints" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent @@ -1112,15 +1397,26 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: List patterns topics with clustered points returns "OK" response - Given operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled + Given a valid "appKeyAuth" key in the system + And operation "ListLLMObsPatternsTopicsWithClusteredPoints" enabled And new "ListLLMObsPatternsTopicsWithClusteredPoints" request And request contains "config_id" parameter from "REPLACE.ME" When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: List versions of an LLM Observability prompt returns "OK" response + Given there is a valid "prompt" in the system + And operation "ListLLMObsPromptVersions" enabled + And new "ListLLMObsPromptVersions" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Lock LLM Observability dataset draft state returns "Bad Request" response - Given operation "LockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "LockLLMObsDatasetDraftState" enabled And new "LockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1129,7 +1425,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Lock LLM Observability dataset draft state returns "Not Found" response - Given operation "LockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "LockLLMObsDatasetDraftState" enabled And new "LockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1138,7 +1435,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Lock LLM Observability dataset draft state returns "OK" response - Given operation "LockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "LockLLMObsDatasetDraftState" enabled And new "LockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1147,7 +1445,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Push events for an LLM Observability experiment returns "Accepted" response - Given operation "CreateLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperimentEvents" enabled And new "CreateLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"metrics": [{"assessment": "pass", "error": {}, "label": "faithfulness", "metric_type": "score", "span_id": "span-7a1b2c3d", "tags": [], "timestamp_ms": 1705314600000}], "spans": [{"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "duration": 1500000000, "meta": {"error": {"message": "Model response timed out", "stack": "Traceback (most recent call last):\n File \"main.py\", line 10, in \n response = model.generate(input)\n File \"model.py\", line 45, in generate\n raise TimeoutError(\"Model response timed out\")\nTimeoutError: Model response timed out", "type": "TimeoutError"}, "input": null, "output": null}, "name": "llm_call", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751", "span_id": "span-7a1b2c3d", "start_ns": 1705314600000000000, "status": "ok", "tags": [], "trace_id": "abc123def456"}]}, "type": "events"}} @@ -1156,7 +1455,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Push events for an LLM Observability experiment returns "Bad Request" response - Given operation "CreateLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperimentEvents" enabled And new "CreateLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"metrics": [{"assessment": "pass", "error": {}, "label": "faithfulness", "metric_type": "score", "span_id": "span-7a1b2c3d", "tags": [], "timestamp_ms": 1705314600000}], "spans": [{"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "duration": 1500000000, "meta": {"error": {"message": "Model response timed out", "stack": "Traceback (most recent call last):\n File \"main.py\", line 10, in \n response = model.generate(input)\n File \"model.py\", line 45, in generate\n raise TimeoutError(\"Model response timed out\")\nTimeoutError: Model response timed out", "type": "TimeoutError"}, "input": null, "output": null}, "name": "llm_call", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751", "span_id": "span-7a1b2c3d", "start_ns": 1705314600000000000, "status": "ok", "tags": [], "trace_id": "abc123def456"}]}, "type": "events"}} @@ -1165,7 +1465,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Push events for an LLM Observability experiment returns "Not Found" response - Given operation "CreateLLMObsExperimentEvents" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsExperimentEvents" enabled And new "CreateLLMObsExperimentEvents" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"metrics": [{"assessment": "pass", "error": {}, "label": "faithfulness", "metric_type": "score", "span_id": "span-7a1b2c3d", "tags": [], "timestamp_ms": 1705314600000}], "spans": [{"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "duration": 1500000000, "meta": {"error": {"message": "Model response timed out", "stack": "Traceback (most recent call last):\n File \"main.py\", line 10, in \n response = model.generate(input)\n File \"model.py\", line 45, in generate\n raise TimeoutError(\"Model response timed out\")\nTimeoutError: Model response timed out", "type": "TimeoutError"}, "input": null, "output": null}, "name": "llm_call", "project_id": "a33671aa-24fd-4dcd-9b33-a8ec7dde7751", "span_id": "span-7a1b2c3d", "start_ns": 1705314600000000000, "status": "ok", "tags": [], "trace_id": "abc123def456"}]}, "type": "events"}} @@ -1174,7 +1475,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Restore an LLM Observability dataset version returns "Bad Request" response - Given operation "RestoreLLMObsDatasetVersion" enabled + Given a valid "appKeyAuth" key in the system + And operation "RestoreLLMObsDatasetVersion" enabled And new "RestoreLLMObsDatasetVersion" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1184,7 +1486,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Restore an LLM Observability dataset version returns "Not Found" response - Given operation "RestoreLLMObsDatasetVersion" enabled + Given a valid "appKeyAuth" key in the system + And operation "RestoreLLMObsDatasetVersion" enabled And new "RestoreLLMObsDatasetVersion" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1194,7 +1497,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Restore an LLM Observability dataset version returns "OK" response - Given operation "RestoreLLMObsDatasetVersion" enabled + Given a valid "appKeyAuth" key in the system + And operation "RestoreLLMObsDatasetVersion" enabled And new "RestoreLLMObsDatasetVersion" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1204,7 +1508,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Run an LLM inference returns "Bad Request" response - Given operation "CreateLLMObsIntegrationInference" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsIntegrationInference" enabled And new "CreateLLMObsIntegrationInference" request And request contains "integration" parameter from "REPLACE.ME" And request contains "account_id" parameter from "REPLACE.ME" @@ -1214,7 +1519,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Run an LLM inference returns "OK" response - Given operation "CreateLLMObsIntegrationInference" enabled + Given a valid "appKeyAuth" key in the system + And operation "CreateLLMObsIntegrationInference" enabled And new "CreateLLMObsIntegrationInference" request And request contains "integration" parameter from "REPLACE.ME" And request contains "account_id" parameter from "REPLACE.ME" @@ -1224,7 +1530,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability experimentation entities returns "Bad Request" response - Given operation "SearchLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "SearchLLMObsExperimentation" enabled And new "SearchLLMObsExperimentation" request And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}} When the request is sent @@ -1232,7 +1539,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability experimentation entities returns "OK — all results returned in a single page." response - Given operation "SearchLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "SearchLLMObsExperimentation" enabled And new "SearchLLMObsExperimentation" request And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}} When the request is sent @@ -1240,7 +1548,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability experimentation entities returns "Partial Content — more results are available. Use `meta.after` as the next `page.cursor`." response - Given operation "SearchLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "SearchLLMObsExperimentation" enabled And new "SearchLLMObsExperimentation" request And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 100}}, "type": "experimentation"}} When the request is sent @@ -1248,7 +1557,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability spans returns "Bad Request" response - Given operation "SearchLLMObsSpans" enabled + Given a valid "appKeyAuth" key in the system + And operation "SearchLLMObsSpans" enabled And new "SearchLLMObsSpans" request And body with value {"data": {"attributes": {"filter": {"from": "now-900s", "ml_app": "my-llm-app", "query": "@session_id:abc123def456", "span_id": "abc123def456", "span_kind": "llm", "span_name": "llm_call", "to": "now", "trace_id": "trace-9a8b7c6d5e4f"}, "options": {"include_attachments": true, "time_offset": 0}, "page": {"cursor": "eyJzdGFydCI6MTAwfQ==", "limit": 10}, "sort": "-start_ns"}, "type": "spans"}} When the request is sent @@ -1256,7 +1566,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Search LLM Observability spans returns "OK" response - Given operation "SearchLLMObsSpans" enabled + Given a valid "appKeyAuth" key in the system + And operation "SearchLLMObsSpans" enabled And new "SearchLLMObsSpans" request And body with value {"data": {"attributes": {"filter": {"from": "now-900s", "ml_app": "my-llm-app", "query": "@session_id:abc123def456", "span_id": "abc123def456", "span_kind": "llm", "span_name": "llm_call", "to": "now", "trace_id": "trace-9a8b7c6d5e4f"}, "options": {"include_attachments": true, "time_offset": 0}, "page": {"cursor": "eyJzdGFydCI6MTAwfQ==", "limit": 10}, "sort": "-start_ns"}, "type": "spans"}} When the request is sent @@ -1264,7 +1575,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Simple search experimentation entities returns "Bad Request" response - Given operation "SimpleSearchLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "SimpleSearchLLMObsExperimentation" enabled And new "SimpleSearchLLMObsExperimentation" request And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 50, "number": 1}, "sort": [{"direction": "desc", "field": "created_at"}]}, "type": "experimentation"}} When the request is sent @@ -1272,7 +1584,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Simple search experimentation entities returns "OK" response - Given operation "SimpleSearchLLMObsExperimentation" enabled + Given a valid "appKeyAuth" key in the system + And operation "SimpleSearchLLMObsExperimentation" enabled And new "SimpleSearchLLMObsExperimentation" request And body with value {"data": {"attributes": {"content_preview": {"limit": 500}, "filter": {"include_deleted": false, "is_deleted": false, "query": "my experiment", "scope": ["experiments"], "version": null}, "include": {"user_data": false}, "page": {"limit": 50, "number": 1}, "sort": [{"direction": "desc", "field": "created_at"}]}, "type": "experimentation"}} When the request is sent @@ -1280,7 +1593,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Trigger a patterns run returns "Accepted" response - Given operation "TriggerLLMObsPatterns" enabled + Given a valid "appKeyAuth" key in the system + And operation "TriggerLLMObsPatterns" enabled And new "TriggerLLMObsPatterns" request And body with value {"data": {"attributes": {"config_id": "a7c8d9e0-1234-5678-9abc-def012345678"}, "type": "topic_discovery"}} When the request is sent @@ -1288,7 +1602,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Trigger a patterns run returns "Bad Request" response - Given operation "TriggerLLMObsPatterns" enabled + Given a valid "appKeyAuth" key in the system + And operation "TriggerLLMObsPatterns" enabled And new "TriggerLLMObsPatterns" request And body with value {"data": {"attributes": {"config_id": "a7c8d9e0-1234-5678-9abc-def012345678"}, "type": "topic_discovery"}} When the request is sent @@ -1296,7 +1611,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Trigger a patterns run returns "Not Found" response - Given operation "TriggerLLMObsPatterns" enabled + Given a valid "appKeyAuth" key in the system + And operation "TriggerLLMObsPatterns" enabled And new "TriggerLLMObsPatterns" request And body with value {"data": {"attributes": {"config_id": "a7c8d9e0-1234-5678-9abc-def012345678"}, "type": "topic_discovery"}} When the request is sent @@ -1304,7 +1620,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Unlock LLM Observability dataset draft state returns "Bad Request" response - Given operation "UnlockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "UnlockLLMObsDatasetDraftState" enabled And new "UnlockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1313,7 +1630,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Unlock LLM Observability dataset draft state returns "Not Found" response - Given operation "UnlockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "UnlockLLMObsDatasetDraftState" enabled And new "UnlockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1322,7 +1640,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Unlock LLM Observability dataset draft state returns "OK" response - Given operation "UnlockLLMObsDatasetDraftState" enabled + Given a valid "appKeyAuth" key in the system + And operation "UnlockLLMObsDatasetDraftState" enabled And new "UnlockLLMObsDatasetDraftState" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1331,7 +1650,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update LLM Observability dataset records returns "Bad Request" response - Given operation "UpdateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDatasetRecords" enabled And new "UpdateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1341,7 +1661,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update LLM Observability dataset records returns "Not Found" response - Given operation "UpdateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDatasetRecords" enabled And new "UpdateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1351,7 +1672,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update LLM Observability dataset records returns "OK" response - Given operation "UpdateLLMObsDatasetRecords" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDatasetRecords" enabled And new "UpdateLLMObsDatasetRecords" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1359,9 +1681,47 @@ Feature: LLM Observability When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: Update a specific LLM Observability prompt version returns "Bad Request" response + Given there is a valid "prompt" in the system + And there is a valid "prompt_version" in the system + And a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPromptVersion" enabled + And new "UpdateLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And request contains "version" parameter with value "not-a-number" + And body with value {"data": {"attributes": {"env_ids": [], "labels": []}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: Update a specific LLM Observability prompt version returns "Not Found" response + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPromptVersion" enabled + And new "UpdateLLMObsPromptVersion" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + And request contains "version" parameter with value "1" + And body with value {"data": {"attributes": {"env_ids": [], "labels": []}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Update a specific LLM Observability prompt version returns "OK" response + Given there is a valid "prompt" in the system + And there is a valid "prompt_version" in the system + And a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPromptVersion" enabled + And new "UpdateLLMObsPromptVersion" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And request contains "version" parameter from "prompt_version.data.attributes.version" + And body with value {"data": {"attributes": {"env_ids": [], "labels": []}, "type": "prompt-template-versions"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability annotation queue returns "Bad Request" response - Given operation "UpdateLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} @@ -1370,7 +1730,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability annotation queue returns "Not Found" response - Given operation "UpdateLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} @@ -1379,7 +1740,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability annotation queue returns "OK" response - Given operation "UpdateLLMObsAnnotationQueue" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueue" enabled And new "UpdateLLMObsAnnotationQueue" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}, "description": "Updated description", "name": "Updated queue name"}, "type": "queues"}} @@ -1388,7 +1750,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability dataset returns "Bad Request" response - Given operation "UpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDataset" enabled And new "UpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1398,7 +1761,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability dataset returns "Not Found" response - Given operation "UpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDataset" enabled And new "UpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1408,7 +1772,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability dataset returns "OK" response - Given operation "UpdateLLMObsDataset" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsDataset" enabled And new "UpdateLLMObsDataset" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1418,7 +1783,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability experiment returns "Bad Request" response - Given operation "UpdateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} @@ -1427,7 +1793,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability experiment returns "Not Found" response - Given operation "UpdateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} @@ -1436,7 +1803,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability experiment returns "OK" response - Given operation "UpdateLLMObsExperiment" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsExperiment" enabled And new "UpdateLLMObsExperiment" request And request contains "experiment_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"dataset_id": "9f64e5c7-dc5a-45c8-a17c-1b85f0bec97d", "status": "completed"}, "type": "experiments"}} @@ -1445,7 +1813,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability project returns "Bad Request" response - Given operation "UpdateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsProject" enabled And new "UpdateLLMObsProject" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {}, "type": "projects"}} @@ -1454,7 +1823,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability project returns "Not Found" response - Given operation "UpdateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsProject" enabled And new "UpdateLLMObsProject" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {}, "type": "projects"}} @@ -1463,16 +1833,50 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update an LLM Observability project returns "OK" response - Given operation "UpdateLLMObsProject" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsProject" enabled And new "UpdateLLMObsProject" request And request contains "project_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {}, "type": "projects"}} When the request is sent Then the response status is 200 OK + @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability prompt returns "Bad Request" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPrompt" enabled + And new "UpdateLLMObsPrompt" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And body with value {"data": {"attributes": {}, "type": "prompt-templates"}} + When the request is sent + Then the response status is 400 Bad Request + + @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability prompt returns "Not Found" response + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPrompt" enabled + And new "UpdateLLMObsPrompt" request + And request contains "prompt_id" parameter with value "nonexistent-prompt" + And body with value {"data": {"attributes": {"title": "New title"}, "type": "prompt-templates"}} + When the request is sent + Then the response status is 404 Not Found + + @skip @team:DataDog/ml-observability + Scenario: Update an LLM Observability prompt returns "OK" response + Given there is a valid "prompt" in the system + And a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsPrompt" enabled + And new "UpdateLLMObsPrompt" request + And request contains "prompt_id" parameter from "prompt.data.attributes.prompt_id" + And body with value {"data": {"attributes": {"title": "New title"}, "type": "prompt-templates"}} + When the request is sent + Then the response status is 200 OK + @generated @skip @team:DataDog/ml-observability Scenario: Update annotation queue label schema returns "Bad Request" response - Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled And new "UpdateLLMObsAnnotationQueueLabelSchema" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} @@ -1481,7 +1885,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update annotation queue label schema returns "Not Found" response - Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled And new "UpdateLLMObsAnnotationQueueLabelSchema" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} @@ -1490,7 +1895,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Update annotation queue label schema returns "OK" response - Given operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled + Given a valid "appKeyAuth" key in the system + And operation "UpdateLLMObsAnnotationQueueLabelSchema" enabled And new "UpdateLLMObsAnnotationQueueLabelSchema" request And request contains "queue_id" parameter from "REPLACE.ME" And body with value {"data": {"attributes": {"annotation_schema": {"label_schemas": [{"description": "Rating of the response quality.", "has_assessment": false, "has_reasoning": false, "id": "abc-123", "is_assessment": false, "is_integer": false, "is_required": true, "max": 5.0, "min": 0.0, "name": "quality", "type": "score", "values": ["good", "bad", "neutral"]}]}}, "type": "queues"}} @@ -1499,7 +1905,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Upload records to an LLM Observability dataset returns "Bad Request" response - Given operation "UploadLLMObsDatasetRecordsFile" enabled + Given a valid "appKeyAuth" key in the system + And operation "UploadLLMObsDatasetRecordsFile" enabled And new "UploadLLMObsDatasetRecordsFile" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1508,7 +1915,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Upload records to an LLM Observability dataset returns "Not Found" response - Given operation "UploadLLMObsDatasetRecordsFile" enabled + Given a valid "appKeyAuth" key in the system + And operation "UploadLLMObsDatasetRecordsFile" enabled And new "UploadLLMObsDatasetRecordsFile" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" @@ -1517,7 +1925,8 @@ Feature: LLM Observability @generated @skip @team:DataDog/ml-observability Scenario: Upload records to an LLM Observability dataset returns "OK" response - Given operation "UploadLLMObsDatasetRecordsFile" enabled + Given a valid "appKeyAuth" key in the system + And operation "UploadLLMObsDatasetRecordsFile" enabled And new "UploadLLMObsDatasetRecordsFile" request And request contains "project_id" parameter from "REPLACE.ME" And request contains "dataset_id" parameter from "REPLACE.ME" diff --git a/tests/v2/features/undo.json b/tests/v2/features/undo.json index f7316aecda..9d548b5fff 100644 --- a/tests/v2/features/undo.json +++ b/tests/v2/features/undo.json @@ -4555,6 +4555,69 @@ "type": "idempotent" } }, + "ListLLMObsPrompts": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "CreateLLMObsPrompt": { + "tag": "LLM Observability", + "undo": { + "operationId": "DeleteLLMObsPrompt", + "parameters": [ + { + "name": "prompt_id", + "source": "data.attributes.prompt_id" + } + ], + "type": "unsafe" + } + }, + "DeleteLLMObsPrompt": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, + "GetLLMObsPrompt": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "UpdateLLMObsPrompt": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, + "ListLLMObsPromptVersions": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "CreateLLMObsPromptVersion": { + "tag": "LLM Observability", + "undo": { + "operationId": "TODO", + "parameters": [], + "type": "unsafe" + } + }, + "GetLLMObsPromptVersion": { + "tag": "LLM Observability", + "undo": { + "type": "safe" + } + }, + "UpdateLLMObsPromptVersion": { + "tag": "LLM Observability", + "undo": { + "type": "idempotent" + } + }, "ListLLMObsSpans": { "tag": "LLM Observability", "undo": {