diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 5ac3337..a448431 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -123,7 +123,19 @@ public final class LanguageModelSession: @unchecked Sendable { state.withLock { $0.endResponding() } } } - + + nonisolated func growStreamingTranscript(text: String) { + withMutation(keyPath: \.transcript) { + state.withLock { $0.transcript.appendStreamingResponse(text) } + } + } + + nonisolated func appendTranscriptEntry(_ entry: Transcript.Entry) { + withMutation(keyPath: \.transcript) { + state.withLock { $0.transcript.append(entry) } + } + } + nonisolated private func wrapRespond(_ operation: () async throws -> T) async throws -> T { beginResponding() do { @@ -163,14 +175,8 @@ public final class LanguageModelSession: @unchecked Sendable { textContent = lastSnapshot.rawContent.jsonString } - let responseEntry = Transcript.Entry.response( - Transcript.Response( - assetIDs: [], - segments: [.text(.init(content: textContent))] - ) - ) session.withMutation(keyPath: \.transcript) { - session.state.withLock { $0.transcript.append(responseEntry) } + session.state.withLock { $0.transcript.finalizeStreamedTranscript(textContent, assetIDs: []) } } } } catch { diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index d93c987..1958b3d 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -98,6 +98,8 @@ public struct AnthropicLanguageModel: LanguageModel { /// These parameters are merged into the top-level request JSON, /// allowing you to pass additional options not explicitly modeled. public var extraBody: [String: JSONValue]? + + public var effort: Effort? // MARK: - Nested Types @@ -178,35 +180,90 @@ public struct AnthropicLanguageModel: LanguageModel { } } + /// How much effort the model should put into a task. + /// + /// Docs: https://platform.claude.com/docs/en/build-with-claude/effort + public enum Effort: String, Hashable, Codable, Sendable { + /// Absolute maximum capability with no constraints on token spending. + /// + /// Use Case: Tasks requiring the deepest possible reasoning and most thorough analysis + /// Availability: Claude Fable 5, Claude Mythos 5, Claude Opus 4.8, Claude Mythos Preview, Claude Opus 4.7, Claude Opus 4.6, Claude Sonnet 5, and Claude Sonnet 4.6. + case max + /// Extended capability for long-horizon work. + /// Use Case: Long-running agentic and coding tasks (over 30 minutes) with token budgets in the millions + /// Availability: Claude Fable 5, Claude Mythos 5, Claude Opus 4.8, Claude Opus 4.7, and Claude Sonnet 5. + case extraHigh = "xHigh" + /// High capability. Equivalent to not setting the parameter. + /// Use Case: Complex reasoning, difficult coding problems, agentic tasks + /// Availability: All Models + case high + /// Balanced approach with moderate token savings. + /// Use Case: Agentic tasks that require a balance of speed, cost, and performance + /// Availability: All Models + case medium + /// Most efficient. Significant token savings with some capability reduction. + /// Use Case: Simpler tasks that need the best speed and lowest costs, like subagents + /// Availability: All Models + case low + } + /// Configuration for extended thinking. public struct Thinking: Hashable, Codable, Sendable { /// The type of thinking to use. public var type: ThinkingType - /// The maximum number of tokens to use for thinking. + /// The maximum number of tokens to use for thinking. Nil when `type` = `.adaptive`. /// /// This budget is the maximum number of tokens the model can use for its /// internal reasoning process. Larger budgets can improve response quality /// for complex tasks but increase latency and cost. - public var budgetTokens: Int + public var budgetTokens: Int? + + /// How thinking should be displayed. + public var display: ThinkingDisplay? /// The type of thinking mode. public enum ThinkingType: String, Hashable, Codable, Sendable { /// Enables extended thinking. case enabled + /// Enables adaptive thinking. + case adaptive + } + + /// How thinking should be returned during generation. + public enum ThinkingDisplay: String, Hashable, Codable, Sendable { + /// Thinking will be summarized. + case summarized + /// No thoughts will be returned. + case omitted } enum CodingKeys: String, CodingKey { case type case budgetTokens = "budget_tokens" + case display } /// Creates a thinking configuration. - /// - /// - Parameter budgetTokens: The maximum number of tokens to use for thinking. - public init(budgetTokens: Int) { - self.type = .enabled + /// + /// - Parameters: + /// - type: The type of thinking to perform. + /// - budgetTokens: The maximum number of tokens to use for thinking. Only required when `type` == `.enabled`. + /// - display: The display type for thoughts. + public init(type: ThinkingType, budgetTokens: Int?, display: ThinkingDisplay?) { + self.type = type self.budgetTokens = budgetTokens + self.display = display + } + + /// Convenience function for enabling adaptive thinking on supported models. + public static func adaptive(display: ThinkingDisplay?) -> Thinking { + return Thinking.init(type: .adaptive, budgetTokens: nil, display: display) + } + + /// Convenience function for enabling thinking with a token budget on supported models. + public static func enabled(budgetTokens: Int, display: ThinkingDisplay?) -> Thinking { + return Thinking.init(type: .enabled, budgetTokens: budgetTokens, display: display) } } @@ -241,7 +298,8 @@ public struct AnthropicLanguageModel: LanguageModel { toolChoice: ToolChoice? = nil, thinking: Thinking? = nil, serviceTier: ServiceTier? = nil, - extraBody: [String: JSONValue]? = nil + extraBody: [String: JSONValue]? = nil, + effort: Effort? = nil ) { self.topP = topP self.topK = topK @@ -251,6 +309,7 @@ public struct AnthropicLanguageModel: LanguageModel { self.thinking = thinking self.serviceTier = serviceTier self.extraBody = extraBody + self.effort = effort } } /// The reason the model is unavailable. @@ -324,73 +383,99 @@ public struct AnthropicLanguageModel: LanguageModel { let anthropicTools: [AnthropicTool] = try session.tools.map { tool in try convertToolToAnthropicFormat(tool) } - + let responseSchema = type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) - let params = try createMessageParams( - model: model, - system: nil, - messages: session.transcript.toAnthropicMessages(), - tools: anthropicTools.isEmpty ? nil : anthropicTools, - responseSchema: responseSchema, - options: options - ) - - let body = try JSONEncoder().encode(params) - - let message: AnthropicMessageResponse = try await httpSession.fetch( - .post, - url: url, - headers: headers, - body: body - ) - + var entries: [Transcript.Entry] = [] + var runningText = "" + var messages: [AnthropicMessage] = session.transcript.toAnthropicMessages() + + // Loop until no more tool calls are found. + while true { + let params = try createMessageParams( + model: model, + system: nil, + messages: messages, + tools: anthropicTools.isEmpty ? nil : anthropicTools, + responseSchema: responseSchema, + options: options + ) + + let body = try JSONEncoder().encode(params) + + let message: AnthropicMessageResponse = try await httpSession.fetch( + .post, + url: url, + headers: headers, + body: body + ) - // Handle tool calls, if present - let toolUses: [AnthropicToolUse] = message.content.compactMap { block in - if case .toolUse(let u) = block { return u } - return nil - } + // Append to messages for future response loops. + messages.append(AnthropicMessage(role: .assistant, content: message.content)) + + // Handle tool calls, if present + let toolUses: [AnthropicToolUse] = message.content.compactMap { content in + if case .toolUse(let u) = content { return u } + return nil + } - if !toolUses.isEmpty { - let resolution = try await resolveToolUses(toolUses, session: session) - switch resolution { - case .stop(let calls): - if !calls.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(calls))) - } - let empty = try emptyResponseContent(for: type) - return LanguageModelSession.Response( - content: empty.content, - rawContent: empty.rawContent, - transcriptEntries: ArraySlice(entries) - ) - case .invocations(let invocations): - if !invocations.isEmpty { - entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) - for invocation in invocations { - entries.append(.toolOutput(invocation.output)) + if !toolUses.isEmpty { + let resolution = try await resolveToolUses(toolUses, session: session) + switch resolution { + case .stop(let calls): + if !calls.isEmpty { + entries.append(.toolCalls(Transcript.ToolCalls(calls))) + } + let empty = try emptyResponseContent(for: type) + return LanguageModelSession.Response( + content: empty.content, + rawContent: empty.rawContent, + transcriptEntries: ArraySlice(entries) + ) + case .invocations(let invocations): + if !invocations.isEmpty { + var toolResultBlocks: [AnthropicContent] = [] + + for invocation in invocations { + entries.append(.toolOutput(invocation.output)) + toolResultBlocks.append( + .toolResult( + AnthropicToolResult( + toolUseId: invocation.call.id, + content: convertSegmentsToAnthropicContent(invocation.output.segments) + ) + ) + ) + } + + messages.append(AnthropicMessage(role: .user, content: toolResultBlocks)) + entries.append(.toolCalls(Transcript.ToolCalls(invocations.map(\.call)))) + + continue // Keep going through the loop } } } - } - let text = message.content.compactMap { block -> String? in - switch block { - case .text(let t): return t.text - default: return nil - } - }.joined() + // If we make it here, no tools were called and we can assume this is the last message. + runningText = message.content.compactMap { block -> String? in + switch block { + case .text(let t): return t.text + default: return nil + } + }.joined() + + break // Break the loop + } if type == String.self { return LanguageModelSession.Response( - content: text as! Content, - rawContent: GeneratedContent(text), + content: runningText as! Content, + rawContent: GeneratedContent(runningText), transcriptEntries: ArraySlice(entries) ) } - let rawContent = try GeneratedContent(json: text) + let rawContent = try GeneratedContent(json: runningText) let content = try Content(rawContent) return LanguageModelSession.Response( content: content, @@ -398,6 +483,20 @@ public struct AnthropicLanguageModel: LanguageModel { transcriptEntries: ArraySlice(entries) ) } + + + struct ContentAccumulationBlocks: Hashable, Codable, Sendable { + enum Kind: Hashable, Codable, Sendable { case toolUse, thinking, text } + + var kind: Kind + var text: String + var partialJSON: String? + + // Used by tool use content blocks + var id: String? + var name: String? + var signature: String? + } public func streamResponse( within session: LanguageModelSession, @@ -419,60 +518,198 @@ public struct AnthropicLanguageModel: LanguageModel { try convertToolToAnthropicFormat(tool) } - let responseSchema = - type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) - var params = try createMessageParams( - model: model, - system: nil, - messages: session.transcript.toAnthropicMessages(), - tools: anthropicTools.isEmpty ? nil : anthropicTools, - responseSchema: responseSchema, - options: options - ) - params["stream"] = .bool(true) - - let body = try JSONEncoder().encode(params) - - // Stream server-sent events from Anthropic API - let events: AsyncThrowingStream = - httpSession - .fetchEventStream( - .post, - url: url, - headers: headers, - body: body - ) - - var accumulatedText = "" + let responseSchema = type == String.self ? nil : try convertSchemaToAnthropicFormat(Content.generationSchema) let expectsStructuredResponse = type != String.self + + var messages: [AnthropicMessage] = session.transcript.toAnthropicMessages() + + while true { + var accumulatedText: String = "" + var stopReason: String? = nil + + let params = try createMessageParams( + model: model, + system: nil, + messages: messages, + tools: anthropicTools.isEmpty ? nil : anthropicTools, + responseSchema: responseSchema, + options: options, + stream: true + ) + + let body = try JSONEncoder().encode(params) + + // Stream server-sent events from Anthropic API + let events: AsyncThrowingStream = + httpSession + .fetchEventStream( + .post, + url: url, + headers: headers, + body: body + ) + + // Accumulating content blocks keyed by their index. + var contentBlocks: [Int: ContentAccumulationBlocks] = [:] + + eventStream: for try await event in events { + switch event { + case .contentBlockStart(let start): + // Create blocks at their index + switch start.contentBlock.type { + case "tool_use": + contentBlocks[start.index] = ContentAccumulationBlocks( + kind: .toolUse, + text: start.contentBlock.text ?? "", + id: start.contentBlock.id, + name: start.contentBlock.name + ) + case "thinking": + contentBlocks[start.index] = ContentAccumulationBlocks( + kind: .thinking, + text: start.contentBlock.text ?? "" + ) + default: + contentBlocks[start.index] = ContentAccumulationBlocks( + kind: .text, + text: start.contentBlock.text ?? "" + ) + } + case .contentBlockDelta(let delta): + switch delta.delta { + case .textDelta(let textDelta): + // Accumulate text delta for streaming + // Make sure the block has even been started. + guard contentBlocks[delta.index] != nil else { continue } + + // Set default text + if contentBlocks[delta.index]?.text == nil { + contentBlocks[delta.index]?.text = "" + } + + contentBlocks[delta.index]?.text += textDelta.text + accumulatedText += textDelta.text + + // Grow the observable transcript so a Transcript-driven UI updates live. + session.growStreamingTranscript(text: accumulatedText) + + // Send text back normally + if expectsStructuredResponse { + if let snapshot: LanguageModelSession.ResponseStream.Snapshot = + try? partialSnapshot(from: accumulatedText) + { + continuation.yield(snapshot) + } + } else { + let raw = GeneratedContent(accumulatedText) + let content: Content.PartiallyGenerated = (accumulatedText as! Content) + .asPartiallyGenerated() + continuation.yield(.init(content: content, rawContent: raw)) + } + case .inputJsonDelta(let jsonDelta): + if contentBlocks[delta.index]?.partialJSON == nil { + contentBlocks[delta.index]?.partialJSON = "" + } + + contentBlocks[delta.index]?.partialJSON? += jsonDelta.partialJson + case .thinkingDelta(let thinkingDelta): + contentBlocks[delta.index]?.text += thinkingDelta.thinking + case .signatureDelta(let signatureDelta): + if contentBlocks[delta.index]?.signature == nil { + contentBlocks[delta.index]?.signature = "" + } - for try await event in events { - switch event { - case .contentBlockDelta(let delta): - if case .textDelta(let textDelta) = delta.delta { - accumulatedText += textDelta.text - - if expectsStructuredResponse { - if let snapshot: LanguageModelSession.ResponseStream.Snapshot = - try? partialSnapshot(from: accumulatedText) - { - continuation.yield(snapshot) + contentBlocks[delta.index]?.signature? += signatureDelta.signature + case .ignored: + break + } + case .messageDelta(let messageDelta): + stopReason = messageDelta.delta.stopReason + case .messageStop: + // Need to use a label, otherwise this would break out of the switch. + break eventStream + case .ping, .ignored, .messageStart, .contentBlockStop: + continue + } + } + + // Assemble assistant content from the streamed content blocks + var assistantContent: [AnthropicContent] = [] + var toolUses: [AnthropicToolUse] = [] + + for block in contentBlocks.sorted(by: { $0.key < $1.key }).map(\.value) { + switch block.kind { + case .text: + assistantContent.append( + AnthropicContent.text(AnthropicText(text: block.text)) + ) + case .thinking: + // Ensure there is a signature. Needed for claude to reconstruct the thought on the server. + guard let signature = block.signature else { continue } + + assistantContent.append( + AnthropicContent.thinking(AnthropicThinking(thinking: block.text, signature: signature)) + ) + case .toolUse: + guard let id = block.id, let name = block.name, let jsonString = block.partialJSON else { continue } + guard let json = fromPartialJSON(jsonString) else { continue } + + let toolUse = AnthropicToolUse(id: id, name: name, input: json) + assistantContent.append(AnthropicContent.toolUse(toolUse)) + toolUses.append(toolUse) + } + } + + messages.append(AnthropicMessage(role: .assistant, content: assistantContent)) + + // Process the tool calls + var appendedToolResults = false + if !toolUses.isEmpty { + let resolution = try await resolveToolUses(toolUses, session: session) + switch resolution { + case .stop(let calls): + if !calls.isEmpty { + session.appendTranscriptEntry(.toolCalls(Transcript.ToolCalls(calls))) + } + continuation.finish() + return + case .invocations(let invocations): + if !invocations.isEmpty { + var toolResultBlocks: [AnthropicContent] = [] + + // Need to append tool calls before tool results + session.appendTranscriptEntry( + .toolCalls(Transcript.ToolCalls(invocations.map(\.call))) + ) + + for invocation in invocations { + // Save the tool outputs into the transcript. + session.appendTranscriptEntry(.toolOutput(invocation.output)) + + toolResultBlocks.append( + .toolResult( + AnthropicToolResult( + toolUseId: invocation.call.id, + content: convertSegmentsToAnthropicContent(invocation.output.segments) + ) + ) + ) } - } else { - let raw = GeneratedContent(accumulatedText) - let content: Content.PartiallyGenerated = (accumulatedText as! Content) - .asPartiallyGenerated() - continuation.yield(.init(content: content, rawContent: raw)) + + messages.append(AnthropicMessage(role: .user, content: toolResultBlocks)) + appendedToolResults = true } } - case .messageStop: - continuation.finish() - return - case .messageStart, .contentBlockStart, .contentBlockStop, .messageDelta, .ping, .ignored: + } + + // Continue only if we responded with tool call results, if we didn't the turn is complete. + if stopReason == "tool_use" && appendedToolResults { + continue + } else { break } } - + continuation.finish() } catch { continuation.finish(throwing: error) @@ -506,7 +743,8 @@ private func createMessageParams( messages: [AnthropicMessage], tools: [AnthropicTool]?, responseSchema: JSONSchema?, - options: GenerationOptions + options: GenerationOptions, + stream: Bool? = nil ) throws -> [String: JSONValue] { var params: [String: JSONValue] = [ "model": .string(model), @@ -577,11 +815,31 @@ private func createMessageParams( params["tool_choice"] = .object(["type": .string("none")]) } } + if let effort = customOptions.effort { + // If output_config was previously set during the response schema options, we need to append insert into that dictionary instead of replacing it. + if let output_config = params["output_config"], var object = output_config.objectValue { + object["effort"] = .string(effort.rawValue) + params["output_config"] = .object(object) + } else { + params["output_config"] = .object( + [ + "effort": .string(effort.rawValue) + ] + ) + } + } if let thinking = customOptions.thinking { - params["thinking"] = .object([ - "type": .string(thinking.type.rawValue), - "budget_tokens": .int(thinking.budgetTokens), - ]) + var thinkingObject: [String: JSONValue] = [ + "type": .string(thinking.type.rawValue) + ] + if let budget = thinking.budgetTokens { + thinkingObject["budget_tokens"] = .int(budget) + } + if let display = thinking.display { + thinkingObject["display"] = .string(display.rawValue) + } + + params["thinking"] = .object(thinkingObject) } if let serviceTier = customOptions.serviceTier { params["service_tier"] = .string(serviceTier.rawValue) @@ -595,6 +853,10 @@ private func createMessageParams( } } + if let stream { + params["stream"] = .bool(stream) + } + return params } @@ -773,6 +1035,19 @@ private func fromGeneratedContent(_ content: GeneratedContent) throws -> [String return dict } +private func fromPartialJSON(_ json: String) -> [String: JSONValue]? { + let content = json.trimmingCharacters(in: .whitespacesAndNewlines) + guard !content.isEmpty else { return [:] } + guard let data = content.data(using: .utf8) else { return nil } + guard let jsonValue = try? JSONDecoder().decode(JSONValue.self, from: data) else { return nil } + + guard case .object(let dict) = jsonValue else { + return nil + } + return dict +} + + // MARK: - Supporting Types extension Transcript { @@ -813,6 +1088,8 @@ extension Transcript { ) ) } + + print("Tool use block \(toolUseBlocks)") messages.append( .init( role: .assistant, @@ -821,6 +1098,7 @@ extension Transcript { ) case .toolOutput(let toolOutput): // Add user message with tool result + print("tool output \(toolOutput.id)") messages.append( .init( role: .user, @@ -903,10 +1181,12 @@ private enum AnthropicContent: Codable, Sendable { private struct AnthropicThinking: Codable, Sendable { let type: String let thinking: String + let signature: String - init(thinking: String) { + init(thinking: String, signature: String) { self.type = "thinking" self.thinking = thinking + self.signature = signature } } @@ -1107,6 +1387,11 @@ private enum AnthropicStreamEvent: Codable, Sendable { struct ContentBlock: Codable, Sendable { let type: String let text: String? + + // Used by tool use content blocks. + let id: String? + let name: String? + let input: [String: JSONValue]? } } @@ -1118,6 +1403,8 @@ private enum AnthropicStreamEvent: Codable, Sendable { enum Delta: Codable, Sendable { case textDelta(TextDelta) case inputJsonDelta(InputJsonDelta) + case thinkingDelta(ThinkingDelta) + case signatureDelta(SignatureDelta) case ignored enum CodingKeys: String, CodingKey { case type } @@ -1131,6 +1418,10 @@ private enum AnthropicStreamEvent: Codable, Sendable { self = .textDelta(try TextDelta(from: decoder)) case "input_json_delta": self = .inputJsonDelta(try InputJsonDelta(from: decoder)) + case "thinking_delta": + self = .thinkingDelta(try ThinkingDelta(from: decoder)) + case "signature_delta": + self = .signatureDelta(try SignatureDelta(from: decoder)) default: self = .ignored } @@ -1140,6 +1431,8 @@ private enum AnthropicStreamEvent: Codable, Sendable { switch self { case .textDelta(let delta): try delta.encode(to: encoder) case .inputJsonDelta(let delta): try delta.encode(to: encoder) + case .thinkingDelta(let delta): try delta.encode(to: encoder) + case .signatureDelta(let delta): try delta.encode(to: encoder) case .ignored: var container = encoder.container(keyedBy: CodingKeys.self) try container.encode("ignored", forKey: .type) @@ -1160,6 +1453,20 @@ private enum AnthropicStreamEvent: Codable, Sendable { case partialJson = "partial_json" } } + + struct ThinkingDelta: Codable, SendableMetatype { + let type: String + let thinking: String + } + + /// Cryptographic signature for a completed thinking block. + /// + /// Emitted at the end of a thinking block, even when ``CustomGenerationOptions/Thinking/display`` is set to `omitted`. + /// The signature must be preserved verbatim for thought to be recovered in the transcript. Otherwise the Claude API will throw out any text provided in thinking blocks. + struct SignatureDelta: Codable, Sendable { + let type: String + let signature: String + } } } diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index dcd1fad..9716623 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -21,6 +21,66 @@ public struct Transcript: Sendable, Equatable, Codable { mutating func append(contentsOf newEntries: S) where S: Sequence, S.Element == Entry { entries.append(contentsOf: newEntries) } + + mutating private func replace(index: Int, with entry: Entry) { + entries[index] = entry + } + + /// Updates a transcript with temporary text that is being streamed from a model. + /// Appends the assistant response to the end of entries, if the last entry is a response then that response is updated with the newest streamed text. + /// + /// - Parameter text: The text to update the transcript with. + mutating func appendStreamingResponse(_ text: String) { + // Make sure the last entry in the transcript is a response. If it is not, create a new response and append it to the end of the transcript. + guard case .response(var response) = entries.last else { + append(Entry.response(Response(assetIDs: [], segments: [ + Transcript.Segment.text(Transcript.TextSegment(content: text)) + ]))) + return + } + + // If the last segment in the last response is text, replace it with the new content. + if case .text(let last)? = response.segments.last { + // Keep the same ID as the last segment. + response.segments[response.segments.count - 1] = Transcript.Segment.text(Transcript.TextSegment(id: last.id, content: text)) + } else { + response.segments.append(Transcript.Segment.text(Transcript.TextSegment(content: text))) + } + + // Replace the latest entry with the one we just updated. + replace(index: entries.count - 1, with: .response(response)) + } + + /// Replaces the trailing response entry's text with the final text, or appends a new response entry if the last entry isn't a response. + /// Prevents streamed responses from having duplicate entries on completion. + /// + /// - Parameters: + /// - text: The text to replace the final response with. + /// - assetIDs: The assetIDs for the response. + mutating func finalizeStreamedTranscript(_ text: String, assetIDs: [String]) { + // Make sure the last entry in the transcript is a response. If it is not, create a new response and append it to the end of the transcript. + guard case .response(let response) = entries.last else { + append(Entry.response(Response(assetIDs: assetIDs, segments: [ + Transcript.Segment.text(Transcript.TextSegment(content: text)) + ]))) + return + } + + // If the last segment is text we want to carry its ID over to the new text segment. Otherwise generate a new ID for it. + let id = switch response.segments.last { + case .text(let last): + last.id + default: + UUID().uuidString + } + + let newResponse: Entry = Entry.response(Response(id: response.id, assetIDs: assetIDs, segments: [ + Transcript.Segment.text(Transcript.TextSegment(id: id, content: text)) + ])) + + replace(index: entries.count - 1, with: newResponse) + } + /// An entry in a transcript. public enum Entry: Sendable, Identifiable, Equatable, Codable { diff --git a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift index 705418e..b0fd1bd 100644 --- a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift @@ -11,7 +11,7 @@ private struct AnthropicStructuredForecast { var temperatureCelsius: Int } -@Suite("AnthropicLanguageModel", .enabled(if: anthropicAPIKey?.isEmpty == false)) +@Suite("AnthropicLanguageModel", .serialized, .enabled(if: anthropicAPIKey?.isEmpty == false)) struct AnthropicLanguageModelTests { let model = AnthropicLanguageModel( apiKey: anthropicAPIKey!, @@ -84,6 +84,39 @@ struct AnthropicLanguageModelTests { #expect(!snapshots.last!.rawContent.jsonString.isEmpty) #expect(!(snapshots.last!.content.summary ?? "").isEmpty) } + + @Test func streamingTranscript() async throws { + let session = LanguageModelSession(model: model) + + let stream = session.streamResponse(to: "Say 'Hello' slowly") + + var snapshots: [LanguageModelSession.ResponseStream.Snapshot] = [] + for try await snapshot in stream { + snapshots.append(snapshot) + + // Make sure the snapshot is also in the transcript. + let hasTranscriptEntry = session.transcript.contains(where: { entry in + switch entry { + case .response(let response): + return response.segments.contains { segment in + switch segment { + case .text(let text): + return text.content.contains(snapshot.content) + default: + return false + } + } + default: + return false + } + }) + #expect(hasTranscriptEntry, "Expected the string snapshot to also appear in the transcript during streaming.") + } + + #expect(!snapshots.isEmpty) + #expect(!snapshots.last!.rawContent.jsonString.isEmpty) + } + @Test func withGenerationOptions() async throws { let session = LanguageModelSession(model: model) @@ -125,9 +158,9 @@ struct AnthropicLanguageModelTests { @Test func withTools() async throws { let weatherTool = WeatherTool() let session = LanguageModelSession(model: model, tools: [weatherTool]) - + let response = try await session.respond(to: "How's the weather in San Francisco?") - + var foundToolOutput = false for case let .toolOutput(toolOutput) in response.transcriptEntries { #expect(!toolOutput.id.isEmpty) @@ -137,6 +170,44 @@ struct AnthropicLanguageModelTests { #expect(foundToolOutput) } + + @Test func streamWithTools() async throws { + let weatherTool = WeatherTool() + let session = LanguageModelSession(model: model, tools: [weatherTool]) + + let stream = session.streamResponse(to: "How's the weather in San Francisco?") + + var snapshots: [LanguageModelSession.ResponseStream.Snapshot] = [] + + var toolAppearedInTranscript: Bool = false + var toolResponseAppearedInTranscript: Bool = false + + for try await snapshot in stream { + snapshots.append(snapshot) + + for entry in session.transcript { + switch entry { + case .toolCalls: + toolAppearedInTranscript = true + case .toolOutput: + toolResponseAppearedInTranscript = true + default: break + } + } + } + + #expect(toolAppearedInTranscript, "Expected a tool call to appear in the transcript during streaming.") + #expect(toolResponseAppearedInTranscript, "Expected a tool output to appear in the transcript during streaming.") + + var foundToolOutput = false + for case let .toolOutput(toolOutput) in session.transcript { + #expect(!toolOutput.id.isEmpty) + #expect(toolOutput.toolName == "getWeather") + foundToolOutput = true + } + #expect(foundToolOutput, "Expected the 'getWeather' tool to exist in the final transcript.") + } + @Test func multimodalWithImageURL() async throws { let session = LanguageModelSession(model: model) let response = try await session.respond( diff --git a/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift b/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift index 3a4a812..b0f5366 100644 --- a/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift +++ b/Tests/AnyLanguageModelTests/CustomGenerationOptionsTests.swift @@ -152,7 +152,7 @@ struct AnthropicCustomOptionsTests { stopSequences: ["END", "STOP"], metadata: .init(userID: "user-123"), toolChoice: .auto, - thinking: .init(budgetTokens: 1024), + thinking: .init(type: .enabled, budgetTokens: 1024, display: .summarized), serviceTier: .priority, extraBody: ["custom_param": .string("value")] ) @@ -187,7 +187,7 @@ struct AnthropicCustomOptionsTests { stopSequences: ["END"], metadata: .init(userID: "user-123"), toolChoice: .tool(name: "my_tool"), - thinking: .init(budgetTokens: 2048), + thinking: .init(type: .enabled, budgetTokens: 2048, display: .summarized), serviceTier: .standard ) @@ -218,7 +218,7 @@ struct AnthropicCustomOptionsTests { topP: 0.9, topK: 40, stopSequences: ["END"], - thinking: .init(budgetTokens: 4096) + thinking: .init(type: .enabled, budgetTokens: 4096, display: .omitted) ) let retrieved = options[custom: AnthropicLanguageModel.self] @@ -226,6 +226,8 @@ struct AnthropicCustomOptionsTests { #expect(retrieved?.topK == 40) #expect(retrieved?.stopSequences == ["END"]) #expect(retrieved?.thinking?.budgetTokens == 4096) + #expect(retrieved?.thinking?.type == .enabled) + #expect(retrieved?.thinking?.display == .omitted) } @Test func metadataCodable() throws { @@ -292,7 +294,7 @@ struct AnthropicCustomOptionsTests { } @Test func thinkingCodable() throws { - let thinking = AnthropicLanguageModel.CustomGenerationOptions.Thinking(budgetTokens: 8192) + let thinking = AnthropicLanguageModel.CustomGenerationOptions.Thinking(type: .enabled, budgetTokens: 8192, display: .omitted) let encoder = JSONEncoder() let data = try encoder.encode(thinking) @@ -302,6 +304,8 @@ struct AnthropicCustomOptionsTests { #expect(json.contains("budget_tokens")) #expect(json.contains("8192")) #expect(json.contains("enabled")) + #expect(json.contains("display")) + #expect(json.contains("omitted")) let decoded = try JSONDecoder().decode( AnthropicLanguageModel.CustomGenerationOptions.Thinking.self,