Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions pyrit/prompt_target/openai/openai_chat_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,11 +296,16 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message |
request: The original request MessagePiece.

Returns:
None if valid, does not return Message for content filter (handled by _check_content_filter).
None when the response is valid and should be constructed normally. Returns an empty
response ``Message`` (with ``error="empty"``) when the response was truncated at the
token limit (``finish_reason="length"``) before producing any visible output, so the
run continues instead of raising. Content filter responses are handled separately by
``_check_content_filter``.

Raises:
PyritException: For unexpected response structures or finish reasons.
EmptyResponseException: When the API returns an empty response.
EmptyResponseException: When the API returns an empty response that was not caused by
token-limit truncation.
"""
# Check for missing choices
if not hasattr(response, "choices") or not response.choices:
Expand All @@ -320,6 +325,25 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message |
# Check for at least one valid response type
has_content, has_audio, has_tool_calls = self._detect_response_content(choice.message)

# A "length" finish_reason means the model hit max_completion_tokens. Reasoning models spend
# tokens on hidden reasoning before emitting visible output, so a low limit can truncate the
# answer or leave it empty. This may be a deliberate configuration, so warn instead of raising.
if finish_reason == "length":
logger.warning(
"The response was truncated because it reached the token limit (finish_reason='length'). "
"Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a "
"low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens "
"if you expected complete content."
)
if not (has_content or has_audio or has_tool_calls):
return construct_response_from_request(
request=request,
response_text_pieces=[""],
response_type="text",
error="empty",
)
return None

if not (has_content or has_audio or has_tool_calls):
logger.error("The chat returned an empty response (no content, audio, or tool_calls).")
raise EmptyResponseException(
Expand Down
21 changes: 18 additions & 3 deletions tests/unit/prompt_target/target/test_openai_chat_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -1039,11 +1039,26 @@ def test_validate_response_success_stop(target: OpenAIChatTarget, dummy_text_mes
assert result is None


def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece):
"""Test _validate_response passes for valid length response."""
def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog):
"""Test _validate_response passes for a truncated response that still has content, and warns."""
mock_response = create_mock_completion(content="Hello", finish_reason="length")
result = target._validate_response(mock_response, dummy_text_message_piece)
with caplog.at_level(logging.WARNING):
result = target._validate_response(mock_response, dummy_text_message_piece)
assert result is None
assert "finish_reason='length'" in caplog.text


def test_validate_response_length_empty_returns_empty_response(
target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog
):
"""Test _validate_response returns a graceful empty response (no raise) when truncated before any output."""
mock_response = create_mock_completion(content="", finish_reason="length")
with caplog.at_level(logging.WARNING):
result = target._validate_response(mock_response, dummy_text_message_piece)
assert isinstance(result, Message)
assert result.message_pieces[0].original_value == ""
assert result.message_pieces[0].response_error == "empty"
assert "finish_reason='length'" in caplog.text


def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece):
Expand Down
Loading