Skip to content

fix(anthropic,gemini): preserve provider-specific reasoning signatures across turns #1153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tbille wants to merge 3 commits into
base: main
Choose a base branch
from
+475 −9
Open

fix(anthropic,gemini): preserve provider-specific reasoning signatures across turns #1153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5d53f3f
fix(anthropic,gemini): preserve provider-specific reasoning signature…
tbille Jun 30, 2026
ea606fc
fix(anthropic): guard None/list assistant content when replaying thin…
tbille Jul 1, 2026
5646d95
refactor(gemini): extract shared _thought_signature_extra_content helper
tbille Jul 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 70 additions & 3 deletions src/any_llm/providers/anthropic/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,45 @@ def _is_tool_call(message: dict[str, Any]) -> bool:
return message["role"] == "assistant" and message.get("tool_calls") is not None


def _extract_anthropic_thinking_signature(message: dict[str, Any]) -> str | None:
"""Extract the encrypted thinking signature stored on a message's extra_content, if any."""
extra_content = message.get("extra_content")
if isinstance(extra_content, dict) and isinstance(anthropic_extra := extra_content.get("anthropic"), dict):
signature = anthropic_extra.get("signature")
if isinstance(signature, str):
return signature
return None


def _extract_reasoning_text(message: dict[str, Any]) -> str:
"""Extract the plain-text reasoning content from a message, regardless of its shape.

``reasoning`` may be a plain string (the OpenAI-wire-compatible serialized form) or a
``{"content": str}`` dict, depending on how the caller constructed the message.
"""
reasoning = message.get("reasoning")
if isinstance(reasoning, str):
return reasoning
if isinstance(reasoning, dict) and isinstance(content := reasoning.get("content"), str):
return content
return ""


def _build_anthropic_thinking_block(message: dict[str, Any]) -> dict[str, Any] | None:
"""Reconstruct an Anthropic ``thinking`` content block for replay across turns.

When extended thinking is enabled, Anthropic requires the ``thinking`` block (including
its encrypted ``signature``) to be passed back unmodified on subsequent turns, e.g.
alongside tool results. Without it, the model loses its original reasoning trace, which
can lead to degraded or repeated reasoning. See
https://docs.claude.com/en/docs/build-with-claude/extended-thinking#preserving-thinking-blocks
"""
signature = _extract_anthropic_thinking_signature(message)
if signature is None:
return None
return {"type": "thinking", "thinking": _extract_reasoning_text(message), "signature": signature}


def _convert_content_for_anthropic(content: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Convert content blocks from OpenAI format to Anthropic format.
- Parse the "content" field block by block
Expand Down Expand Up @@ -122,9 +161,11 @@ def _convert_messages_for_anthropic(messages: list[dict[str, Any]]) -> tuple[str
# See https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/overview#tool-use-examples
if _is_tool_call(message):
# Convert ALL tool calls from the assistant message
tool_use_blocks = []
content_blocks: list[dict[str, Any]] = []
if thinking_block := _build_anthropic_thinking_block(message):
content_blocks.append(thinking_block)
for tool_call in message["tool_calls"]:
tool_use_blocks.append(
content_blocks.append(
{
"type": "tool_use",
"id": tool_call["id"],
Expand All @@ -134,7 +175,7 @@ def _convert_messages_for_anthropic(messages: list[dict[str, Any]]) -> tuple[str
)
message = {
"role": "assistant",
"content": tool_use_blocks,
"content": content_blocks,
}
elif message["role"] == "tool":
# Use tool_call_id from the message itself
Expand All @@ -157,6 +198,20 @@ def _convert_messages_for_anthropic(messages: list[dict[str, Any]]) -> tuple[str
"role": "user",
"content": [tool_result],
}
elif message["role"] == "assistant" and (thinking_block := _build_anthropic_thinking_block(message)):
# existing_content may be None (a reasoning-only turn with no text/tool_calls),
# a plain string, or a list of content blocks.
existing_content = message.get("content")
content_blocks = [thinking_block]
if isinstance(existing_content, str):
if existing_content:
content_blocks.append({"type": "text", "text": existing_content})
elif isinstance(existing_content, list):
content_blocks.extend(existing_content)
message = {
"role": "assistant",
"content": content_blocks,
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

if "content" in message and isinstance(message["content"], list):
message["content"] = _convert_content_for_anthropic(message["content"])
Expand Down Expand Up @@ -212,6 +267,11 @@ def _create_openai_chunk_from_anthropic_chunk(chunk: Any, model_id: str) -> Chat
}
elif chunk.delta.type == "thinking_delta":
delta = {"reasoning": {"content": chunk.delta.thinking}}
elif chunk.delta.type == "signature_delta":
# The encrypted signature of the thinking block. Must be preserved unmodified
# and passed back to Anthropic on subsequent turns (e.g. alongside tool results)
# to maintain reasoning continuity. See https://docs.claude.com/en/docs/build-with-claude/extended-thinking
delta = {"extra_content": {"anthropic": {"signature": chunk.delta.signature}}}

elif isinstance(chunk, ContentBlockStopEvent):
if hasattr(chunk, "content_block") and chunk.content_block.type == "tool_use":
Expand Down Expand Up @@ -254,6 +314,7 @@ def _convert_response(response: Message) -> ChatCompletion:
content_parts: list[str] = []
tool_calls: list[ChatCompletionMessageFunctionToolCall | ChatCompletionMessageToolCall] = []
reasoning_content: str | None = None
thinking_signature: str | None = None
for content_block in response.content:
if content_block.type == "text":
content_parts.append(content_block.text)
Expand All @@ -273,6 +334,11 @@ def _convert_response(response: Message) -> ChatCompletion:
reasoning_content = content_block.thinking
else:
reasoning_content += content_block.thinking
# The encrypted signature must be preserved and replayed unmodified on
# subsequent turns (e.g. alongside tool results) to maintain reasoning
# continuity. See https://docs.claude.com/en/docs/build-with-claude/extended-thinking
if content_block.signature:
thinking_signature = content_block.signature
else:
msg = f"Unsupported content block type: {content_block.type}"
raise ValueError(msg)
Expand All @@ -282,6 +348,7 @@ def _convert_response(response: Message) -> ChatCompletion:
content="".join(content_parts),
reasoning=Reasoning(content=reasoning_content) if reasoning_content else None,
tool_calls=cast("list[ChatCompletionMessageToolCallType] | None", tool_calls or None),
extra_content={"anthropic": {"signature": thinking_signature}} if thinking_signature else None,
)

cache_read = response.usage.cache_read_input_tokens or 0
Expand Down
24 changes: 19 additions & 5 deletions src/any_llm/providers/gemini/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,18 @@ def _extract_usage_dict(response: types.GenerateContentResponse) -> dict[str, An
return usage


def _thought_signature_extra_content(part: Any) -> dict[str, Any] | None:
"""Build the OpenAI-compatible extra_content payload for a Gemini thought_signature, if present.

Shared by both the non-streaming (_convert_response_to_response_dict) and streaming
(_create_openai_chunk_from_google_chunk) conversion paths so they stay in sync.
"""
thought_signature = getattr(part, "thought_signature", None)
if thought_signature is not None and isinstance(thought_signature, bytes):
return {"google": {"thought_signature": base64.b64encode(thought_signature).decode("utf-8")}}
return None
Comment on lines +279 to +288

@coderabbitai coderabbitai Bot Jul 1, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the target file around the reported lines.
sed -n '240,320p' src/any_llm/providers/gemini/utils.py

# Find where types.Part is imported and used in this file.
rg -n "types\.Part|from .* import .*types|import .*types" src/any_llm/providers/gemini/utils.py

# Locate the definition of Part in the repository, if present.
fd -a "types.py" src any_llm . | sed -n '1,40p'
rg -n "class Part|def .*thought_signature|thought_signature" src any_llm . -g '!**/.git/**'

Repository: mozilla-ai/any-llm

Length of output: 11963

🌐 Web query:

google genai types.Part thought_signature attribute documented Part thought_signature bytes

💡 Result:

In the Google Gen AI ecosystem, thought_signature is a field used to maintain reasoning continuity and state for models that support thinking [1][2]. It acts as an opaque, encrypted representation of the model's internal reasoning process [2][3]. Regarding the Part object and the thought_signature data type: 1. Data Type: The thought_signature attribute is generally handled as a string [4][5]. While some SDKs (notably Go) may define the type internally as []byte [6], it is functionally an opaque string provided by the API [4]. Developers are advised that if an SDK uses []byte, standard JSON marshaling might incorrectly encode the value (e.g., base64 encoding a bypass string), which can cause the signature to become non-functional [6]. The Python and JavaScript SDKs typically handle this as a string, allowing for straightforward pass-through of the value [4][6][5]. 2. Purpose and Usage: - Preservation: It is used to preserve the Gemini reasoning state in multi-turn and multi-step conversations, particularly when function calling is involved [1][2]. - Requirements: You must include all Part objects from previous messages exactly as they were returned by the model [2][3]. You should not merge, modify, or omit parts that contain these signatures, as this can degrade reasoning quality or trigger 400 errors in strict validation scenarios (such as with the first functionCall part in a turn) [1][2]. - Non-Function Call Parts: Gemini may also return thought_signatures in the final Part of a response (e.g., text parts) even when no function call occurs [1][2]. While returning these is not strictly enforced with a blocking error, it is recommended to ensure high-quality reasoning and performance [1][2]. 3. Documentation: The field is documented as an optional attribute within the Part interface (e.g., Part.thoughtSignature) [4][5]. It is explicitly described as an opaque signature designed to be reused in subsequent requests to maintain context [4][7][5].

Citations:

🏁 Script executed:

#!/bin/bash
set -euo pipefail

sed -n '80,150p' src/any_llm/types/completion.py
sed -n '860,930p' tests/unit/providers/test_gemini_provider.py
sed -n '980,1065p' tests/unit/providers/test_gemini_provider.py

Repository: mozilla-ai/any-llm

Length of output: 8962

🌐 Web query:

python-genai Part thought_signature attribute type bytes docs

💡 Result:

In the Google Gen AI Python SDK (google-genai), the thought_signature attribute of the Part class is defined as bytes [1][2]. Key details regarding this attribute include: - Purpose: It is an opaque, encrypted representation of the model's internal reasoning (thought) process [3][4][5]. It is required to maintain reasoning continuity across multi-turn interactions, particularly when using thinking-enabled models [3][6][7]. - API Behavior: When Gemini generates content (such as function calls or text), it may include a thought_signature in the Part [3][6]. If the model response contains a functionCall, including the thought_signature when sending the conversation history back is mandatory; omitting it often results in a 400 error [3][6][8]. - SDK Implementation: The SDK models this field within the Part class as Optional[bytes] [1][2]. While the underlying REST API often represents this signature as a string (frequently base64-encoded) [9][5], the Python SDK types it as bytes [1][2]. - Handling: When using standard chat history features or appending the full model response to the history, the official SDK typically handles the attachment of the thought_signature automatically behind the scenes [6]. However, users have reported issues where the signature may not be correctly passed back in certain scenarios (such as specific streaming patterns or multi-turn function calls), requiring careful management of the content history [10][8]. Documentation on this field can be found in the SDK source code types definitions [11][2] and the Gemini API documentation regarding thinking models [3][6][4].

Citations:

Use types.Part here and access thought_signature directly src/any_llm/providers/gemini/utils.py:279

The helper only receives google.genai.types.Part objects, and thought_signature is a declared field there, so part: types.Part plus part.thought_signature is the cleaner type-safe form.

🧰 Tools
🪛 Ruff (0.15.20)

[warning] 279-279: Dynamically typed expressions (typing.Any) are disallowed in part

(ANN401)

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/any_llm/providers/gemini/utils.py` around lines 279 - 288, The helper
`_thought_signature_extra_content` is using a generic `Any` lookup for
`thought_signature` even though it only handles `google.genai.types.Part`
objects. Update the function signature to use `types.Part` and read
`part.thought_signature` directly, keeping the existing bytes check and base64
payload shape so both response conversion paths remain consistent.

Sources: Coding guidelines, Linters/SAST tools



def _convert_response_to_response_dict(response: types.GenerateContentResponse) -> dict[str, Any]:
response_dict = {
"id": "google_genai_response",
Expand Down Expand Up @@ -315,11 +327,8 @@ def _convert_response_to_response_dict(response: types.GenerateContentResponse)
}

# Include thought_signature if present (OpenAI compatibility format)
thought_signature = getattr(part, "thought_signature", None)
if thought_signature is not None and isinstance(thought_signature, bytes):
tool_call_dict["extra_content"] = {
"google": {"thought_signature": base64.b64encode(thought_signature).decode("utf-8")}
}
if extra_content := _thought_signature_extra_content(part):
tool_call_dict["extra_content"] = extra_content

tool_calls_list.append(tool_call_dict)
elif getattr(part, "text", None):
Expand Down Expand Up @@ -405,6 +414,10 @@ def _create_openai_chunk_from_google_chunk(
for key, value in args.items():
args_dict[key] = value

# Include thought_signature if present (OpenAI compatibility format), mirroring
# the non-streaming conversion in _convert_response_to_response_dict.
extra_content = _thought_signature_extra_content(part)

Comment thread
coderabbitai[bot] marked this conversation as resolved.
tool_calls_list.append(
ChoiceDeltaToolCall(
index=len(tool_calls_list),
Expand All @@ -414,6 +427,7 @@ def _create_openai_chunk_from_google_chunk(
name=function_call.name,
arguments=json.dumps(args_dict),
),
extra_content=extra_content,
)
)
elif part.text:
Expand Down
28 changes: 27 additions & 1 deletion src/any_llm/types/completion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,15 @@ def _serialize(self) -> str:
class ChatCompletionMessage(OpenAIChatCompletionMessage):
reasoning: Reasoning | None = None
annotations: list[dict[str, Any]] | None = None # type: ignore[assignment]
extra_content: dict[str, Any] | None = None
"""Provider-specific metadata that needs to be preserved across multi-turn conversations.

For example, Anthropic's extended thinking requires the encrypted ``signature`` of a
``thinking`` block to be passed back unmodified alongside subsequent tool calls.

Example extra_content structure for Anthropic:
{"anthropic": {"signature": "<encrypted-signature>"}}
"""


class Choice(OpenAIChoice):
Expand All @@ -84,8 +93,26 @@ class ParsedChatCompletion(ChatCompletion, Generic[ContentType]):
choices: list[ParsedChoice[ContentType]] # type: ignore[assignment]


class ChoiceDeltaToolCall(OpenAIChoiceDeltaToolCall):
"""Streaming counterpart of ``ChatCompletionMessageFunctionToolCall``.

Adds the same ``extra_content`` field so provider-specific tool-call metadata (e.g.
Gemini's ``thought_signature``) can be carried on streaming deltas, not just on the
final non-streaming tool call.
"""

extra_content: dict[str, Any] | None = None


class ChoiceDelta(OpenAIChoiceDelta):
reasoning: Reasoning | None = None
tool_calls: list[ChoiceDeltaToolCall] | None = None # type: ignore[assignment]
extra_content: dict[str, Any] | None = None
"""Streaming counterpart of ``ChatCompletionMessage.extra_content``.

Carries provider-specific metadata (e.g. Anthropic's thinking block ``signature``)
that arrives as part of a streaming delta rather than the final message.
"""


class ChunkChoice(OpenAIChunkChoice):
Expand Down Expand Up @@ -118,7 +145,6 @@ class ChatCompletionMessageFunctionToolCall(OpenAIChatCompletionMessageFunctionT
CreateEmbeddingResponse = OpenAICreateEmbeddingResponse
Embedding = OpenAIEmbedding
Usage = OpenAIUsage
ChoiceDeltaToolCall = OpenAIChoiceDeltaToolCall
ChoiceDeltaToolCallFunction = OpenAIChoiceDeltaToolCallFunction

ReasoningEffort = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "auto"]
Expand Down
Loading
Loading