From 513373e9b9008116c5b0bb611efa573d51c04465 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:46:33 -0700 Subject: [PATCH] Warn instead of raising on reasoning-model token truncation When a reasoning model hits max_completion_tokens, the API returns finish_reason='length' with empty visible content. Previously this raised EmptyResponseException (misleadingly reported as 'Status Code: 204') and triggered the 10x retry storm. _validate_response now logs a warning explaining that reasoning models consume tokens on hidden reasoning, and returns a graceful empty response (error='empty') instead of raising, so runs continue. Non-length empty responses still raise as before. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../openai/openai_chat_target.py | 28 +++++++++++++++++-- .../target/test_openai_chat_target.py | 21 ++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pyrit/prompt_target/openai/openai_chat_target.py b/pyrit/prompt_target/openai/openai_chat_target.py index 9374ea73b8..cdcd28a480 100644 --- a/pyrit/prompt_target/openai/openai_chat_target.py +++ b/pyrit/prompt_target/openai/openai_chat_target.py @@ -296,11 +296,16 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | request: The original request MessagePiece. Returns: - None if valid, does not return Message for content filter (handled by _check_content_filter). + None when the response is valid and should be constructed normally. Returns an empty + response ``Message`` (with ``error="empty"``) when the response was truncated at the + token limit (``finish_reason="length"``) before producing any visible output, so the + run continues instead of raising. Content filter responses are handled separately by + ``_check_content_filter``. Raises: PyritException: For unexpected response structures or finish reasons. - EmptyResponseException: When the API returns an empty response. + EmptyResponseException: When the API returns an empty response that was not caused by + token-limit truncation. """ # Check for missing choices if not hasattr(response, "choices") or not response.choices: @@ -320,6 +325,25 @@ def _validate_response(self, response: Any, request: MessagePiece) -> Message | # Check for at least one valid response type has_content, has_audio, has_tool_calls = self._detect_response_content(choice.message) + # A "length" finish_reason means the model hit max_completion_tokens. Reasoning models spend + # tokens on hidden reasoning before emitting visible output, so a low limit can truncate the + # answer or leave it empty. This may be a deliberate configuration, so warn instead of raising. + if finish_reason == "length": + logger.warning( + "The response was truncated because it reached the token limit (finish_reason='length'). " + "Reasoning models consume tokens on hidden reasoning in addition to the visible answer, so a " + "low max_completion_tokens can truncate or empty the response. Increase max_completion_tokens " + "if you expected complete content." + ) + if not (has_content or has_audio or has_tool_calls): + return construct_response_from_request( + request=request, + response_text_pieces=[""], + response_type="text", + error="empty", + ) + return None + if not (has_content or has_audio or has_tool_calls): logger.error("The chat returned an empty response (no content, audio, or tool_calls).") raise EmptyResponseException( diff --git a/tests/unit/prompt_target/target/test_openai_chat_target.py b/tests/unit/prompt_target/target/test_openai_chat_target.py index dbf1018a1b..39121d797f 100644 --- a/tests/unit/prompt_target/target/test_openai_chat_target.py +++ b/tests/unit/prompt_target/target/test_openai_chat_target.py @@ -1039,11 +1039,26 @@ def test_validate_response_success_stop(target: OpenAIChatTarget, dummy_text_mes assert result is None -def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece): - """Test _validate_response passes for valid length response.""" +def test_validate_response_success_length(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog): + """Test _validate_response passes for a truncated response that still has content, and warns.""" mock_response = create_mock_completion(content="Hello", finish_reason="length") - result = target._validate_response(mock_response, dummy_text_message_piece) + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) assert result is None + assert "finish_reason='length'" in caplog.text + + +def test_validate_response_length_empty_returns_empty_response( + target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece, caplog +): + """Test _validate_response returns a graceful empty response (no raise) when truncated before any output.""" + mock_response = create_mock_completion(content="", finish_reason="length") + with caplog.at_level(logging.WARNING): + result = target._validate_response(mock_response, dummy_text_message_piece) + assert isinstance(result, Message) + assert result.message_pieces[0].original_value == "" + assert result.message_pieces[0].response_error == "empty" + assert "finish_reason='length'" in caplog.text def test_validate_response_no_choices(target: OpenAIChatTarget, dummy_text_message_piece: MessagePiece):