diff --git a/cli/azd/extensions/azure.ai.agents/cspell.yaml b/cli/azd/extensions/azure.ai.agents/cspell.yaml index 5d2d9e0950c..78be91d2fea 100644 --- a/cli/azd/extensions/azure.ai.agents/cspell.yaml +++ b/cli/azd/extensions/azure.ai.agents/cspell.yaml @@ -11,6 +11,9 @@ words: # Voice (prompt-voice) agents - BYOM - Nanami + - pcma + - pcmu + - webrtc # Azure region names - australiaeast - brazilsouth diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go index 5ffe129eeb1..afafade125c 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go @@ -822,23 +822,22 @@ func isDeployed( *errs = append(*errs, fmt.Errorf("read %s: %w", key, err)) return false } - if value != "" { - return true - } - - // Voice agents (kind: prompt-voice) deploy without an agent-version object, - // so they never set AGENT__VERSION. Fall back to the base endpoint - // marker, which every voice deploy writes, so a successfully created voice - // agent is not reported as undeployed. Gate this on the service's actual - // declared kind: a hosted agent whose deploy partially failed can also - // present an empty VERSION with a lingering ENDPOINT, and must stay reported - // as not-deployed. This mirrors the kind gate in + if !isVoice { + return value != "" + } + + // Voice deploys use the base ENDPOINT env var as the completion marker. The + // legacy voice API does not produce AGENT__VERSION, and unified voice + // deploys write VERSION before ENDPOINT to keep ENDPOINT as the final marker. + // Require ENDPOINT for voice even when VERSION is present, otherwise a partial + // env write could be reported as deployed before the callable endpoint was + // persisted. Gate this on the service's actual declared kind: a hosted agent + // whose deploy partially failed can also present an empty VERSION with a + // lingering ENDPOINT, and must stay reported as not-deployed. This mirrors the + // kind gate in // AgentServiceTargetProvider.Endpoints (project package); the two live in // separate packages because project imports nextstep, so a literally shared // helper would create an import cycle. - if !isVoice { - return false - } endpointKey := fmt.Sprintf(agentEndpointVarFormat, serviceKey(serviceName)) endpointValue, err := src.EnvValue(ctx, envName, endpointKey) if err != nil { diff --git a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go index c62408b10bf..7f50fe6f190 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state_test.go @@ -739,10 +739,10 @@ func TestServiceKey(t *testing.T) { } } -// TestIsDeployed_VoiceEndpointFallback verifies that a voice agent — which sets -// only AGENT__NAME and AGENT__ENDPOINT, never AGENT__VERSION — is -// still reported as deployed via the base endpoint marker, while an agent with -// neither version nor endpoint is reported undeployed. +// TestIsDeployed_VoiceEndpointFallback verifies that voice readiness is based +// on the base endpoint marker. Legacy voice agents never set VERSION, while +// unified voice agents set VERSION before ENDPOINT; in both cases ENDPOINT is +// the deploy completion marker. func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { t.Parallel() @@ -757,6 +757,21 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) { values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"}, want: true, }, + { + name: "version set but endpoint missing: undeployed (voice agent partial write)", + values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"}, + isVoice: true, + want: false, + }, + { + name: "version and endpoint set: deployed (unified voice agent)", + values: map[string]string{ + "env1/AGENT_VOICE_SVC_VERSION": "1", + "env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1", + }, + isVoice: true, + want: true, + }, { name: "no version but base endpoint set: deployed (voice agent)", values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "https://x/voice_agents/a"}, diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go index 549f9610f12..ec2ea32be40 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/models.go @@ -360,7 +360,7 @@ const ( // VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz). type VoiceAudioFormat struct { Type string `json:"type"` - Rate int `json:"rate"` + Rate *int `json:"rate,omitempty"` } // VoiceTurnDetection configures server-side voice-activity detection so the @@ -370,32 +370,69 @@ type VoiceTurnDetection struct { Threshold *float64 `json:"threshold,omitempty"` PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"` SilenceDurationMs *int `json:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty"` + AutoTruncate *bool `json:"auto_truncate,omitempty"` } // VoiceTranscription enables user-speech transcription events on the input stream. type VoiceTranscription struct { - Model string `json:"model,omitempty"` + Model string `json:"model,omitempty"` + Language *string `json:"language,omitempty"` + Prompt *string `json:"prompt,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type"` } // VoiceInputConfig is the input (caller -> agent) audio configuration. type VoiceInputConfig struct { - Format *VoiceAudioFormat `json:"format,omitempty"` - TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` - Transcription *VoiceTranscription `json:"transcription,omitempty"` + Format *VoiceAudioFormat `json:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty"` } // VoiceConfig selects the output voice. Type is "openai" for realtime voices // (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural // voices (e.g. "en-US-Ava:DragonHDLatestNeural"). type VoiceConfig struct { - Type string `json:"type"` - Name string `json:"name"` + Type string `json:"type"` + Name string `json:"name"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Locale *string `json:"locale,omitempty"` + Volume *string `json:"volume,omitempty"` } // VoiceOutputConfig is the output (agent -> caller) audio configuration. type VoiceOutputConfig struct { Format *VoiceAudioFormat `json:"format,omitempty"` Voice *VoiceConfig `json:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty"` +} + +// VoiceOutputConfigFlat is the newer Voice Live output shape used by the +// unified /agents voice API in TiP. Older regions still accept/return the +// object-shaped VoiceOutputConfig above. +type VoiceOutputConfigFlat struct { + Format *VoiceAudioFormat `json:"format,omitempty"` + Voice string `json:"voice,omitempty"` + VoiceType string `json:"voice_type,omitempty"` + VoiceLocale string `json:"voice_locale,omitempty"` + Style *string `json:"style,omitempty"` + Pitch *string `json:"pitch,omitempty"` + Rate *string `json:"rate,omitempty"` + Volume *string `json:"volume,omitempty"` + Speed *float64 `json:"speed,omitempty"` } // VoiceAudioConfig bundles the input and output audio configuration. @@ -404,17 +441,54 @@ type VoiceAudioConfig struct { Output *VoiceOutputConfig `json:"output,omitempty"` } +// VoiceAudioConfigFlat bundles voice audio config with the flat output shape. +type VoiceAudioConfigFlat struct { + Input *VoiceInputConfig `json:"input,omitempty"` + Output *VoiceOutputConfigFlat `json:"output,omitempty"` +} + // VoiceAgentDefinition is the data-plane definition body POSTed to the // /voice_agents collection for a declarative (managed) voice agent. Its Kind // is always AgentKindVoice ("voice"). type VoiceAgentDefinition struct { AgentDefinition - ModelType VoiceModelType `json:"model_type"` - Model string `json:"model"` - Instructions string `json:"instructions,omitempty"` - Audio *VoiceAudioConfig `json:"audio,omitempty"` - OutputModalities []string `json:"output_modalities,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfig `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` +} + +// VoiceAgentDefinitionFlat is the voice definition shape aligned with the +// unified /agents TiP API, where audio.output.voice is a string and the voice +// provider details are sibling fields. +type VoiceAgentDefinitionFlat struct { + AgentDefinition + ModelType VoiceModelType `json:"model_type"` + Model string `json:"model"` + Instructions string `json:"instructions,omitempty"` + StructuredInputs map[string]any `json:"structured_inputs,omitempty"` + Audio *VoiceAudioConfigFlat `json:"audio,omitempty"` + OutputModalities []string `json:"output_modalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"tool_choice,omitempty"` + ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"` + MaxOutputTokens any `json:"max_output_tokens,omitempty"` + Include []string `json:"include,omitempty"` } // CreateAgentVersionRequest represents a request to create an agent version diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go index c94ebc69294..fbd6f514ef3 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations.go @@ -167,6 +167,55 @@ func (c *AgentClient) CreateAgent(ctx context.Context, request *CreateAgentReque // header while voice agents remain a preview capability. const voiceAgentsPreviewFeature = "VoiceAgents=V1Preview" +func (c *AgentClient) doVoiceJSONAgentRequest( + ctx context.Context, + method string, + url string, + request any, + overriddenHost string, +) (*AgentObject, error) { + payload, err := json.Marshal(request) + if err != nil { + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := runtime.NewRequest(ctx, method, url) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature) + if overriddenHost != "" { + req.Raw().Header.Set("x-ms-overridden-host", overriddenHost) + } + + if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { + return nil, fmt.Errorf("failed to set request body: %w", err) + } + + resp, err := c.pipeline.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated) { + return nil, runtime.NewResponseError(resp) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + var agent AgentObject + if err := json.Unmarshal(body, &agent); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + + return &agent, nil +} + // CreateVoiceAgent creates a new declarative (managed) voice agent. // // Voice agents live in a separate data-plane collection (/voice_agents), distinct @@ -193,36 +242,36 @@ func (c *AgentClient) CreateVoiceAgent( overriddenHost string, ) (*AgentObject, error) { url := fmt.Sprintf("%s/voice_agents?api-version=%s", c.endpoint, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} - payload, err := json.Marshal(request) - if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) - } - - req, err := runtime.NewRequest(ctx, http.MethodPost, url) +// GetVoiceAgentUnified retrieves a voice agent through the unified /agents +// endpoint with the voice preview opt-in header. Use this instead of GetAgent +// when deciding whether to create or update a prompt voice agent. +func (c *AgentClient) GetVoiceAgentUnified( + ctx context.Context, + agentName string, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) + req, err := runtime.NewRequest(ctx, http.MethodGet, url) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } - // Voice agents are a preview feature; the service rejects the request with - // 403 preview_feature_required unless this opt-in header is present. req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature) - if overriddenHost != "" { req.Raw().Header.Set("x-ms-overridden-host", overriddenHost) } - if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil { - return nil, fmt.Errorf("failed to set request body: %w", err) - } - resp, err := c.pipeline.Do(req) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) } defer resp.Body.Close() - if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated) { + if !runtime.HasStatusCode(resp, http.StatusOK) { return nil, runtime.NewResponseError(resp) } @@ -239,6 +288,31 @@ func (c *AgentClient) CreateVoiceAgent( return &agent, nil } +// CreateVoiceAgentUnified creates a voice agent through the unified /agents +// collection. This path is opt-in while regional rollout is still in progress. +func (c *AgentClient) CreateVoiceAgentUnified( + ctx context.Context, + request *CreateAgentRequest, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents?api-version=%s", c.endpoint, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} + +// UpdateVoiceAgentUnified creates a new version for an existing voice agent +// through the unified /agents/{name} endpoint. +func (c *AgentClient) UpdateVoiceAgentUnified( + ctx context.Context, + agentName string, + request *UpdateAgentRequest, + apiVersion string, + overriddenHost string, +) (*AgentObject, error) { + url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) + return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost) +} + // UpdateAgent updates an existing agent func (c *AgentClient) UpdateAgent(ctx context.Context, agentName string, request *UpdateAgentRequest, apiVersion string) (*AgentObject, error) { url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go index 6eb2effae43..7aacd0d7ba5 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_api/operations_test.go @@ -948,6 +948,73 @@ func TestCreateVoiceAgent_PostsToVoiceCollectionWithPreviewHeader(t *testing.T) require.Contains(t, string(reqBody), `"name":"my-voice"`) } +func TestCreateVoiceAgentUnified_PostsToAgentsWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.CreateVoiceAgentUnified( + t.Context(), + &CreateAgentRequest{Name: "my-voice"}, + AgentEndpointAPIVersion, + "", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "/api/projects/proj/agents", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) +} + +func TestGetVoiceAgentUnified_GetsNamedAgentWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"3"}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.GetVoiceAgentUnified( + t.Context(), + "my-voice", + AgentEndpointAPIVersion, + "regional.hyena.example.com", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Equal(t, "3", agent.Versions.Latest.Version) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodGet, req.Method) + require.Equal(t, "/api/projects/proj/agents/my-voice", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) + require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) +} + +func TestUpdateVoiceAgentUnified_PostsToNamedAgentWithPreviewHeader(t *testing.T) { + body := `{"object":"agent","id":"va-1","name":"my-voice","versions":{"latest":{"version":"2"}}}` + client, transport := newCaptureClient(http.StatusOK, body) + + agent, err := client.UpdateVoiceAgentUnified( + t.Context(), + "my-voice", + &UpdateAgentRequest{}, + AgentEndpointAPIVersion, + "regional.hyena.example.com", + ) + + require.NoError(t, err) + require.Equal(t, "my-voice", agent.Name) + require.Len(t, transport.requests, 1) + req := transport.requests[0] + require.Equal(t, http.MethodPost, req.Method) + require.Equal(t, "/api/projects/proj/agents/my-voice", req.URL.Path) + require.Equal(t, AgentEndpointAPIVersion, req.URL.Query().Get("api-version")) + require.Equal(t, voiceAgentsPreviewFeature, req.Header.Get("Foundry-Features")) + require.Equal(t, "regional.hyena.example.com", req.Header.Get("x-ms-overridden-host")) +} + func TestCreateVoiceAgent_SetsOverriddenHostHeader(t *testing.T) { client, transport := newCaptureClient(http.StatusCreated, `{"name":"my-voice","versions":{"latest":{}}}`) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go index ea3e42f2870..0ea5fe41d89 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map.go @@ -583,10 +583,161 @@ func buildVoiceConfig(name string) *agent_api.VoiceConfig { return &agent_api.VoiceConfig{Type: "azure_standard", Name: trimmed} } +func flatVoiceType(voice *agent_api.VoiceConfig) string { + if voice == nil { + return "" + } + if voice.Type == "azure_standard" { + return "azure-standard" + } + return voice.Type +} + +func normalizeFlatVoiceType(voiceType string) string { + switch strings.TrimSpace(voiceType) { + case "azure_standard": + return "azure-standard" + default: + return strings.TrimSpace(voiceType) + } +} + +func flatVoiceLocale(voice *agent_api.VoiceConfig) string { + if voice == nil || voice.Name == "" || isOpenAIVoice(voice.Name) { + return "" + } + if voice.Locale != nil && strings.TrimSpace(*voice.Locale) != "" { + return strings.TrimSpace(*voice.Locale) + } + parts := strings.SplitN(voice.Name, "-", 3) + if len(parts) < 2 { + return "" + } + return parts[0] + "-" + parts[1] +} + +func defaultVoiceAudioFormat() *agent_api.VoiceAudioFormat { + rate := defaultVoiceAudioRate + return &agent_api.VoiceAudioFormat{Type: defaultVoiceAudioType, Rate: &rate} +} + +func mapVoiceAudioFormat(format *VoiceAudioFormat, fallback *agent_api.VoiceAudioFormat) *agent_api.VoiceAudioFormat { + out := &agent_api.VoiceAudioFormat{} + if fallback != nil { + *out = *fallback + } + if format != nil { + if strings.TrimSpace(format.Type) != "" { + out.Type = strings.TrimSpace(format.Type) + } + if format.Rate != nil { + out.Rate = format.Rate + } + } + return out +} + +func mapVoiceTurnDetection(turnDetection *VoiceTurnDetection) *agent_api.VoiceTurnDetection { + out := &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType} + if turnDetection == nil { + return out + } + if strings.TrimSpace(turnDetection.Type) != "" { + out.Type = strings.TrimSpace(turnDetection.Type) + } + out.Threshold = turnDetection.Threshold + out.PrefixPaddingMs = turnDetection.PrefixPaddingMs + out.SilenceDurationMs = turnDetection.SilenceDurationMs + out.CreateResponse = turnDetection.CreateResponse + out.Eagerness = turnDetection.Eagerness + out.SpeechDurationMs = turnDetection.SpeechDurationMs + out.RemoveFillerWords = turnDetection.RemoveFillerWords + out.InterruptResponse = turnDetection.InterruptResponse + out.Languages = turnDetection.Languages + out.AutoTruncate = turnDetection.AutoTruncate + return out +} + +func mapVoiceTranscription(transcription *VoiceTranscription) *agent_api.VoiceTranscription { + out := &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel} + if transcription == nil { + return out + } + if strings.TrimSpace(transcription.Model) != "" { + out.Model = strings.TrimSpace(transcription.Model) + } + out.Language = transcription.Language + out.Prompt = transcription.Prompt + return out +} + +func mapVoiceConfig(voice *VoiceConfig, fallbackName string) *agent_api.VoiceConfig { + if voice == nil { + return buildVoiceConfig(fallbackName) + } + name := strings.TrimSpace(voice.Name) + if name == "" { + name = fallbackName + } + voiceType := strings.TrimSpace(voice.Type) + if voiceType == "" { + out := buildVoiceConfig(name) + out.Style = voice.Style + out.Pitch = voice.Pitch + out.Rate = voice.Rate + out.Locale = voice.Locale + out.Volume = voice.Volume + return out + } + return &agent_api.VoiceConfig{ + Type: voiceType, + Name: name, + Style: voice.Style, + Pitch: voice.Pitch, + Rate: voice.Rate, + Locale: voice.Locale, + Volume: voice.Volume, + } +} + +func mapVoiceStructuredInputs(inputs map[string]any) map[string]any { + if len(inputs) == 0 { + return nil + } + out := make(map[string]any, len(inputs)) + for name, input := range inputs { + inputMap, ok := input.(map[string]any) + if !ok { + out[name] = input + continue + } + + mapped := maps.Clone(inputMap) + if value, ok := mapped["defaultValue"]; ok { + if _, hasSnakeCase := mapped["default_value"]; !hasSnakeCase { + mapped["default_value"] = value + } + delete(mapped, "defaultValue") + } + out[name] = mapped + } + return out +} + // CreateVoiceAgentAPIRequest builds a CreateAgentRequest for a declarative // voice agent. It translates the authoring kind "prompt-voice" into the // data-plane service kind "voice" and defaults the audio pipeline. func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, false) +} + +// CreateVoiceAgentAPIRequestFlat builds a CreateAgentRequest using the newer +// TiP/unified API flat output voice shape. +func CreateVoiceAgentAPIRequestFlat(voiceAgent VoiceAgent) (*agent_api.CreateAgentRequest, error) { + return createVoiceAgentAPIRequest(voiceAgent, true) +} + +func createVoiceAgentAPIRequest(voiceAgent VoiceAgent, flatOutput bool) (*agent_api.CreateAgentRequest, error) { modelID := "" if voiceAgent.Model != nil { modelID = strings.TrimSpace(voiceAgent.Model.Id) @@ -615,9 +766,80 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe voiceName = *voiceAgent.Voice } - audioFormat := &agent_api.VoiceAudioFormat{ - Type: defaultVoiceAudioType, - Rate: defaultVoiceAudioRate, + inputFormat := defaultVoiceAudioFormat() + outputFormat := defaultVoiceAudioFormat() + turnDetection := mapVoiceTurnDetection(nil) + transcription := mapVoiceTranscription(nil) + var noiseReduction *agent_api.VoiceNoiseReduction + var echoCancellation map[string]any + outputVoice := buildVoiceConfig(voiceName) + var outputSpeed *float64 + if voiceAgent.Audio != nil { + if voiceAgent.Audio.Input != nil { + inputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Input.Format, inputFormat) + if voiceAgent.Audio.Input.NoiseReduction != nil { + noiseReduction = &agent_api.VoiceNoiseReduction{Type: strings.TrimSpace(voiceAgent.Audio.Input.NoiseReduction.Type)} + } + echoCancellation = voiceAgent.Audio.Input.EchoCancellation + turnDetection = mapVoiceTurnDetection(voiceAgent.Audio.Input.TurnDetection) + transcription = mapVoiceTranscription(voiceAgent.Audio.Input.Transcription) + } + if voiceAgent.Audio.Output != nil { + outputFormat = mapVoiceAudioFormat(voiceAgent.Audio.Output.Format, outputFormat) + outputVoice = mapVoiceConfig(voiceAgent.Audio.Output.Voice, voiceName) + outputSpeed = voiceAgent.Audio.Output.Speed + } + } + + outputModalities := []string{"audio"} + if len(voiceAgent.OutputModalities) > 0 { + outputModalities = voiceAgent.OutputModalities + } + + input := &agent_api.VoiceInputConfig{ + Format: inputFormat, + NoiseReduction: noiseReduction, + EchoCancellation: echoCancellation, + TurnDetection: turnDetection, + Transcription: transcription, + } + if flatOutput { + voiceDef := agent_api.VoiceAgentDefinitionFlat{ + AgentDefinition: agent_api.AgentDefinition{ + // Translate authoring kind prompt-voice -> service kind voice. + Kind: agent_api.AgentKindVoice, + }, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), + Audio: &agent_api.VoiceAudioConfigFlat{ + Input: input, + Output: &agent_api.VoiceOutputConfigFlat{ + Format: outputFormat, + Voice: outputVoice.Name, + VoiceType: normalizeFlatVoiceType(flatVoiceType(outputVoice)), + VoiceLocale: flatVoiceLocale(outputVoice), + Style: outputVoice.Style, + Pitch: outputVoice.Pitch, + Rate: outputVoice.Rate, + Volume: outputVoice.Volume, + Speed: outputSpeed, + }, + }, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, + } + + return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) } voiceDef := agent_api.VoiceAgentDefinition{ @@ -625,22 +847,28 @@ func CreateVoiceAgentAPIRequest(voiceAgent VoiceAgent) (*agent_api.CreateAgentRe // Translate authoring kind prompt-voice -> service kind voice. Kind: agent_api.AgentKindVoice, }, - ModelType: modelType, - Model: modelID, - Instructions: instructions, + ModelType: modelType, + Model: modelID, + Instructions: instructions, + StructuredInputs: mapVoiceStructuredInputs(voiceAgent.StructuredInputs), Audio: &agent_api.VoiceAudioConfig{ - Input: &agent_api.VoiceInputConfig{ - Format: audioFormat, - TurnDetection: &agent_api.VoiceTurnDetection{Type: defaultVoiceTurnDetectionType}, - Transcription: &agent_api.VoiceTranscription{Model: defaultVoiceInputTranscriptionModel}, - }, + Input: input, Output: &agent_api.VoiceOutputConfig{ - Format: audioFormat, - Voice: buildVoiceConfig(voiceName), + Format: outputFormat, + Voice: outputVoice, + Speed: outputSpeed, }, }, - OutputModalities: []string{"audio"}, - Store: voiceAgent.Store, + OutputModalities: outputModalities, + Store: voiceAgent.Store, + Tools: voiceAgent.Tools, + Avatar: voiceAgent.Avatar, + Greeting: voiceAgent.Greeting, + Handoff: voiceAgent.Handoff, + ToolChoice: voiceAgent.ToolChoice, + ParallelToolCalls: voiceAgent.ParallelToolCalls, + MaxOutputTokens: voiceAgent.MaxOutputTokens, + Include: voiceAgent.Include, } return createAgentAPIRequest(voiceAgent.AgentDefinition, voiceDef, nil, nil) diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go index 137e9b6e847..be279ddaec7 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/map_voice_test.go @@ -4,6 +4,7 @@ package agent_yaml import ( + "encoding/json" "testing" "azureaiagent/internal/pkg/agents/agent_api" @@ -117,7 +118,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Fatalf("Audio pipeline not populated: %+v", def.Audio) } in := def.Audio.Input - if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate != defaultVoiceAudioRate { + if in.Format == nil || in.Format.Type != defaultVoiceAudioType || in.Format.Rate == nil || + *in.Format.Rate != defaultVoiceAudioRate { t.Errorf("input format = %+v", in.Format) } if in.TurnDetection == nil || in.TurnDetection.Type != defaultVoiceTurnDetectionType { @@ -127,7 +129,8 @@ func TestCreateVoiceAgentAPIRequest_Defaults(t *testing.T) { t.Errorf("transcription = %+v", in.Transcription) } out := def.Audio.Output - if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate != defaultVoiceAudioRate { + if out.Format == nil || out.Format.Type != defaultVoiceAudioType || out.Format.Rate == nil || + *out.Format.Rate != defaultVoiceAudioRate { t.Errorf("output format = %+v", out.Format) } // Default voice is the DragonHD Azure Neural voice. @@ -172,6 +175,193 @@ func TestCreateVoiceAgentAPIRequest_Overrides(t *testing.T) { } } +func TestCreateVoiceAgentAPIRequestFlat_UsesFlatOutputShape(t *testing.T) { + t.Parallel() + voice := "alloy" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + if def.Audio.Output.Voice != "alloy" { + t.Errorf("Voice = %q, want alloy", def.Audio.Output.Voice) + } + if def.Audio.Output.VoiceType != "openai" { + t.Errorf("VoiceType = %q, want openai", def.Audio.Output.VoiceType) + } + if def.Audio.Output.VoiceLocale != "" { + t.Errorf("VoiceLocale = %q, want empty", def.Audio.Output.VoiceLocale) + } +} + +func TestCreateVoiceAgentAPIRequestFlat_AzureVoiceLocale(t *testing.T) { + t.Parallel() + voice := "en-US-Ava:DragonHDLatestNeural" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + def := req.Definition.(agent_api.VoiceAgentDefinitionFlat) + if def.Audio.Output.Voice != voice { + t.Errorf("Voice = %q, want %q", def.Audio.Output.Voice, voice) + } + if def.Audio.Output.VoiceType != "azure-standard" { + t.Errorf("VoiceType = %q, want azure-standard", def.Audio.Output.VoiceType) + } + if def.Audio.Output.VoiceLocale != "en-US" { + t.Errorf("VoiceLocale = %q, want en-US", def.Audio.Output.VoiceLocale) + } +} + +func TestCreateVoiceAgentAPIRequestFlat_MarshalWireShape(t *testing.T) { + t.Parallel() + voice := "en-US-Ava:DragonHDLatestNeural" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-flat"}, + Model: &Model{Id: "gpt-realtime"}, + Voice: &voice, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + definition := wire["definition"].(map[string]any) + audio := definition["audio"].(map[string]any) + output := audio["output"].(map[string]any) + + if got, ok := output["voice"].(string); !ok || got != voice { + t.Fatalf("audio.output.voice = %#v, want string %q", output["voice"], voice) + } + if got := output["voice_type"]; got != "azure-standard" { + t.Fatalf("audio.output.voice_type = %#v, want azure-standard", got) + } + if got := output["voice_locale"]; got != "en-US" { + t.Fatalf("audio.output.voice_locale = %#v, want en-US", got) + } + if _, exists := output["type"]; exists { + t.Fatalf("audio.output.type should not be present in flat wire shape: %#v", output) + } +} + +func TestCreateVoiceAgentAPIRequestFlat_AdvancedSettingsWireShape(t *testing.T) { + t.Parallel() + inRate := 16000 + outRate := 24000 + threshold := 0.6 + speechDurationMs := 120 + createResponse := true + removeFillerWords := true + interruptResponse := true + autoTruncate := true + speed := 1.1 + parallelToolCalls := true + style := "cheerful" + pitch := "+0Hz" + rate := "+0%" + volume := "+0%" + instructions := "You are {{persona}}, a concise voice assistant." + language := "en-US" + prompt := "Contoso terms" + agent := VoiceAgent{ + AgentDefinition: AgentDefinition{Kind: AgentKindPromptVoice, Name: "voice-advanced"}, + Model: &Model{Id: "gpt-realtime"}, + Instructions: &instructions, + StructuredInputs: map[string]any{ + "persona": map[string]any{"description": "Assistant persona", "defaultValue": "Ada"}, + }, + Audio: &VoiceAudio{ + Input: &VoiceAudioInput{ + Format: &VoiceAudioFormat{Type: "audio/pcmu", Rate: &inRate}, + NoiseReduction: &VoiceNoiseReduction{Type: "near_field"}, + EchoCancellation: map[string]any{"type": "server_echo_cancellation", "channels": 1}, + TurnDetection: &VoiceTurnDetection{ + Type: "azure_semantic_vad", + Threshold: &threshold, + SpeechDurationMs: &speechDurationMs, + CreateResponse: &createResponse, + RemoveFillerWords: &removeFillerWords, + InterruptResponse: &interruptResponse, + Languages: []string{"en-US"}, + AutoTruncate: &autoTruncate, + }, + Transcription: &VoiceTranscription{Model: "whisper-1", Language: &language, Prompt: &prompt}, + }, + Output: &VoiceAudioOutput{ + Format: &VoiceAudioFormat{Type: "audio/pcm", Rate: &outRate}, + Voice: &VoiceConfig{ + Type: "azure_standard", Name: "en-US-AvaNeural", Style: &style, + Pitch: &pitch, Rate: &rate, Locale: &language, Volume: &volume, + }, + Speed: &speed, + }, + }, + OutputModalities: []string{"audio", "text"}, + Tools: []map[string]any{{"type": "system", "name": "end_conversation"}}, + Avatar: map[string]any{"type": "video_avatar", "character": "lisa", "output_protocol": "webrtc"}, + Greeting: map[string]any{"type": "template", "text": "Hello {{persona}}"}, + ToolChoice: "auto", + ParallelToolCalls: ¶llelToolCalls, + MaxOutputTokens: "inf", + Include: []string{"item.input_audio_transcription.phrases"}, + } + + req, err := CreateVoiceAgentAPIRequestFlat(agent) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + payload, err := json.Marshal(req) + if err != nil { + t.Fatalf("marshal request: %v", err) + } + + var wire map[string]any + if err := json.Unmarshal(payload, &wire); err != nil { + t.Fatalf("unmarshal payload: %v", err) + } + def := wire["definition"].(map[string]any) + input := def["audio"].(map[string]any)["input"].(map[string]any) + output := def["audio"].(map[string]any)["output"].(map[string]any) + structured := def["structured_inputs"].(map[string]any)["persona"].(map[string]any) + + if structured["default_value"] != "Ada" || structured["defaultValue"] != nil { + t.Fatalf("structured input default was not mapped to wire shape: %#v", structured) + } + if output["voice"] != "en-US-AvaNeural" || output["voice_type"] != "azure-standard" || output["style"] != style { + t.Fatalf("output voice flat shape not mapped: %#v", output) + } + if input["echo_cancellation"].(map[string]any)["type"] != "server_echo_cancellation" { + t.Fatalf("echo cancellation not mapped: %#v", input["echo_cancellation"]) + } + if def["tool_choice"] != "auto" || def["max_output_tokens"] != "inf" { + t.Fatalf("response options not mapped: %#v", def) + } + if len(def["tools"].([]any)) != 1 || def["avatar"].(map[string]any)["character"] != "lisa" { + t.Fatalf("tools/avatar not mapped: %#v", def) + } +} + // TestCreateVoiceAgentAPIRequest_ExplicitManaged verifies that explicitly // setting model_type: managed is accepted (idempotent with the default). func TestCreateVoiceAgentAPIRequest_ExplicitManaged(t *testing.T) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go index 996eae8a561..a90fbe6046d 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse.go @@ -460,6 +460,7 @@ func ValidateAgentDefinition(templateBytes []byte) error { "template.model_type '%s' is not supported; use '%s' or '%s'", agent.ModelType, VoiceModelTypeManaged, VoiceModelTypeSelfDeployed)) } + errors = append(errors, validateVoiceAgentAdvancedConfig(agent)...) } else { errors = append(errors, fmt.Sprintf("failed to unmarshal to VoiceAgent: %v", err)) } @@ -479,6 +480,94 @@ func ValidateAgentDefinition(templateBytes []byte) error { return nil } +func validateVoiceAgentAdvancedConfig(agent VoiceAgent) []string { + var errors []string + for i, modality := range agent.OutputModalities { + if strings.TrimSpace(modality) == "" { + errors = append(errors, fmt.Sprintf("template.output_modalities[%d] must not be blank", i)) + } + } + + if agent.Audio == nil { + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, "")...) + } + transcriptionModel := "" + if agent.Audio.Input != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.input.format", agent.Audio.Input.Format)...) + if nr := agent.Audio.Input.NoiseReduction; nr != nil && strings.TrimSpace(nr.Type) == "" { + errors = append(errors, "template.audio.input.noise_reduction.type must not be blank") + } + if td := agent.Audio.Input.TurnDetection; td != nil { + if strings.TrimSpace(td.Type) == "" { + errors = append(errors, "template.audio.input.turn_detection.type must not be blank") + } + if td.Threshold != nil && (*td.Threshold < 0 || *td.Threshold > 1) { + errors = append(errors, "template.audio.input.turn_detection.threshold must be between 0 and 1") + } + if td.PrefixPaddingMs != nil && *td.PrefixPaddingMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.prefix_padding_ms must be >= 0") + } + if td.SilenceDurationMs != nil && *td.SilenceDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.silence_duration_ms must be >= 0") + } + if td.SpeechDurationMs != nil && *td.SpeechDurationMs < 0 { + errors = append(errors, "template.audio.input.turn_detection.speech_duration_ms must be >= 0") + } + } + if agent.Audio.Input.Transcription != nil { + transcriptionModel = agent.Audio.Input.Transcription.Model + } + } + if agent.Audio.Output != nil { + errors = append(errors, validateVoiceAudioFormat("template.audio.output.format", agent.Audio.Output.Format)...) + if voice := agent.Audio.Output.Voice; voice != nil { + if strings.TrimSpace(voice.Type) == "" { + errors = append(errors, "template.audio.output.voice.type must not be blank") + } + if strings.TrimSpace(voice.Name) == "" { + errors = append(errors, "template.audio.output.voice.name must not be blank") + } + } + if speed := agent.Audio.Output.Speed; speed != nil && (*speed < 0.25 || *speed > 1.5) { + errors = append(errors, "template.audio.output.speed must be between 0.25 and 1.5") + } + } + return append(errors, validateVoiceIncludeTranscriptionCompatibility(agent, transcriptionModel)...) +} + +func validateVoiceIncludeTranscriptionCompatibility(agent VoiceAgent, transcriptionModel string) []string { + if !slices.Contains(agent.Include, "item.input_audio_transcription.phrases") { + return nil + } + model := strings.TrimSpace(transcriptionModel) + if model == "" { + model = defaultVoiceInputTranscriptionModel + } + if model == "azure-speech" || model == "azure-fast-transcription" { + return nil + } + return []string{ + "template.include item.input_audio_transcription.phrases requires template.audio.input.transcription.model to be azure-speech or azure-fast-transcription", + } +} + +func validateVoiceAudioFormat(path string, format *VoiceAudioFormat) []string { + if format == nil { + return nil + } + var errors []string + formatType := strings.TrimSpace(format.Type) + if formatType == "" { + errors = append(errors, path+".type must not be blank") + } else if formatType != "audio/pcm" && formatType != "audio/pcmu" && formatType != "audio/pcma" { + errors = append(errors, path+".type must be 'audio/pcm', 'audio/pcmu', or 'audio/pcma'") + } + if format.Rate != nil && *format.Rate <= 0 { + errors = append(errors, path+".rate must be greater than 0") + } + return errors +} + // Validate that the agent name matches the expected deployable format func ValidateAgentName(name string) error { if name == "" { diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go index 6aba8a368dd..b348d867651 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/parse_voice_test.go @@ -118,3 +118,24 @@ model_type: unsupported t.Fatalf("expected invalid model_type error, got: %v", err) } } +func TestValidateAgentDefinition_PromptVoice_InvalidIncludeTranscriptionModel(t *testing.T) { + yamlContent := []byte(` +kind: prompt-voice +name: voice-agent +model: + id: gpt-realtime +audio: + input: + transcription: + model: whisper-1 +include: + - item.input_audio_transcription.phrases +`) + err := ValidateAgentDefinition(yamlContent) + if err == nil { + t.Fatal("expected include/transcription validation error") + } + if !strings.Contains(err.Error(), "azure-speech") || !strings.Contains(err.Error(), "azure-fast-transcription") { + t.Fatalf("expected transcription model guidance in error, got: %v", err) + } +} diff --git a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go index 46fffca9e6a..ffa35d057db 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go +++ b/cli/azd/extensions/azure.ai.agents/internal/pkg/agents/agent_yaml/yaml.go @@ -214,9 +214,98 @@ type VoiceAgent struct { // Voice is the output voice name (e.g. "en-US-Ava:DragonHDLatestNeural" for // an Azure Neural voice, or "alloy" for an OpenAI realtime voice). Voice *string `json:"voice,omitempty" yaml:"voice,omitempty"` + // StructuredInputs declares template inputs used by voice instructions and greeting. + StructuredInputs map[string]any `json:"structuredInputs,omitempty" yaml:"structured_inputs,omitempty"` + // Audio customizes the input and output voice pipeline. Missing fields keep azd defaults. + Audio *VoiceAudio `json:"audio,omitempty" yaml:"audio,omitempty"` + // OutputModalities declares response modalities such as audio, text, animation, or avatar. + OutputModalities []string `json:"outputModalities,omitempty" yaml:"output_modalities,omitempty"` // Store toggles server-side logging (transcript + per-turn audio). Optional; // the service defaults to false when omitted. Store *bool `json:"store,omitempty" yaml:"store,omitempty"` + // Tools are passed through to the prompt voice service. Supported direct tool + // types include function, mcp, system, and toolbox. + Tools []map[string]any `json:"tools,omitempty" yaml:"tools,omitempty"` + // Avatar customizes voice avatar output for services that support it. + Avatar map[string]any `json:"avatar,omitempty" yaml:"avatar,omitempty"` + // Greeting configures initial greeting behavior for services that support it. + Greeting map[string]any `json:"greeting,omitempty" yaml:"greeting,omitempty"` + // Handoff configures voice handoff behavior for services that support it. + Handoff map[string]any `json:"handoff,omitempty" yaml:"handoff,omitempty"` + // ToolChoice configures service tool choice behavior, such as auto/none/required. + ToolChoice any `json:"toolChoice,omitempty" yaml:"tool_choice,omitempty"` + // ParallelToolCalls toggles parallel tool calls. + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty" yaml:"parallel_tool_calls,omitempty"` + // MaxOutputTokens limits response output tokens. Use an integer or service-supported string such as "inf". + MaxOutputTokens any `json:"maxOutputTokens,omitempty" yaml:"max_output_tokens,omitempty"` + // Include requests additional service response fields. + Include []string `json:"include,omitempty" yaml:"include,omitempty"` +} + +// VoiceAudio bundles optional prompt voice input/output audio overrides. +type VoiceAudio struct { + Input *VoiceAudioInput `json:"input,omitempty" yaml:"input,omitempty"` + Output *VoiceAudioOutput `json:"output,omitempty" yaml:"output,omitempty"` +} + +// VoiceAudioInput customizes caller-to-agent audio. +type VoiceAudioInput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + NoiseReduction *VoiceNoiseReduction `json:"noiseReduction,omitempty" yaml:"noise_reduction,omitempty"` + EchoCancellation map[string]any `json:"echoCancellation,omitempty" yaml:"echo_cancellation,omitempty"` + TurnDetection *VoiceTurnDetection `json:"turnDetection,omitempty" yaml:"turn_detection,omitempty"` + Transcription *VoiceTranscription `json:"transcription,omitempty" yaml:"transcription,omitempty"` +} + +// VoiceAudioOutput customizes agent-to-caller audio. +type VoiceAudioOutput struct { + Format *VoiceAudioFormat `json:"format,omitempty" yaml:"format,omitempty"` + Voice *VoiceConfig `json:"voice,omitempty" yaml:"voice,omitempty"` + Speed *float64 `json:"speed,omitempty" yaml:"speed,omitempty"` +} + +// VoiceAudioFormat describes an audio stream format. +type VoiceAudioFormat struct { + Type string `json:"type" yaml:"type"` + Rate *int `json:"rate,omitempty" yaml:"rate,omitempty"` +} + +// VoiceNoiseReduction configures input audio noise reduction. +type VoiceNoiseReduction struct { + Type string `json:"type" yaml:"type"` +} + +// VoiceTurnDetection configures server-side turn detection. +type VoiceTurnDetection struct { + Type string `json:"type" yaml:"type"` + Threshold *float64 `json:"threshold,omitempty" yaml:"threshold,omitempty"` + PrefixPaddingMs *int `json:"prefixPaddingMs,omitempty" yaml:"prefix_padding_ms,omitempty"` + SilenceDurationMs *int `json:"silenceDurationMs,omitempty" yaml:"silence_duration_ms,omitempty"` + CreateResponse *bool `json:"createResponse,omitempty" yaml:"create_response,omitempty"` + Eagerness *string `json:"eagerness,omitempty" yaml:"eagerness,omitempty"` + SpeechDurationMs *int `json:"speechDurationMs,omitempty" yaml:"speech_duration_ms,omitempty"` + RemoveFillerWords *bool `json:"removeFillerWords,omitempty" yaml:"remove_filler_words,omitempty"` + InterruptResponse *bool `json:"interruptResponse,omitempty" yaml:"interrupt_response,omitempty"` + Languages []string `json:"languages,omitempty" yaml:"languages,omitempty"` + AutoTruncate *bool `json:"autoTruncate,omitempty" yaml:"auto_truncate,omitempty"` +} + +// VoiceTranscription configures input transcription. +type VoiceTranscription struct { + Model string `json:"model,omitempty" yaml:"model,omitempty"` + Language *string `json:"language,omitempty" yaml:"language,omitempty"` + Prompt *string `json:"prompt,omitempty" yaml:"prompt,omitempty"` +} + +// VoiceConfig selects the output voice. +type VoiceConfig struct { + Type string `json:"type" yaml:"type"` + Name string `json:"name" yaml:"name"` + Style *string `json:"style,omitempty" yaml:"style,omitempty"` + Pitch *string `json:"pitch,omitempty" yaml:"pitch,omitempty"` + Rate *string `json:"rate,omitempty" yaml:"rate,omitempty"` + Locale *string `json:"locale,omitempty" yaml:"locale,omitempty"` + Volume *string `json:"volume,omitempty" yaml:"volume,omitempty"` } // ContainerResources represents the resource allocation for a containerized agent. diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go index 26b5a06f874..1625d9a7637 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/agent_definition.go @@ -138,35 +138,68 @@ type AgentDefinitionInline struct { // Voice-agent fields (kind: prompt-voice). All omitempty so container/ // workflow entries are byte-for-byte unchanged. - ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` - Model *agent_yaml.Model `json:"model,omitempty"` - Instructions *string `json:"instructions,omitempty"` - Voice *string `json:"voice,omitempty"` - Store *bool `json:"store,omitempty"` + ModelType agent_yaml.VoiceModelType `json:"modelType,omitempty"` + Model *agent_yaml.Model `json:"model,omitempty"` + Instructions *string `json:"instructions,omitempty"` + Voice *string `json:"voice,omitempty"` + StructuredInputs map[string]any `json:"structuredInputs,omitempty"` + Audio *agent_yaml.VoiceAudio `json:"audio,omitempty"` + OutputModalities []string `json:"outputModalities,omitempty"` + Store *bool `json:"store,omitempty"` + Tools []map[string]any `json:"tools,omitempty"` + Avatar map[string]any `json:"avatar,omitempty"` + Greeting map[string]any `json:"greeting,omitempty"` + Handoff map[string]any `json:"handoff,omitempty"` + ToolChoice any `json:"toolChoice,omitempty"` + ParallelToolCalls *bool `json:"parallelToolCalls,omitempty"` + MaxOutputTokens any `json:"maxOutputTokens,omitempty"` + Include []string `json:"include,omitempty"` } // voiceAgentDefinitionToInline projects a VoiceAgent into the inline definition // written to azure.yaml. Voice agents carry no container/image/code config. func voiceAgentDefinitionToInline(va agent_yaml.VoiceAgent) AgentDefinitionInline { return AgentDefinitionInline{ - AgentDefinition: va.AgentDefinition, - ModelType: va.ModelType, - Model: va.Model, - Instructions: va.Instructions, - Voice: va.Voice, - Store: va.Store, + AgentDefinition: va.AgentDefinition, + ModelType: va.ModelType, + Model: va.Model, + Instructions: va.Instructions, + Voice: va.Voice, + StructuredInputs: va.StructuredInputs, + Audio: va.Audio, + OutputModalities: va.OutputModalities, + Store: va.Store, + Tools: va.Tools, + Avatar: va.Avatar, + Greeting: va.Greeting, + Handoff: va.Handoff, + ToolChoice: va.ToolChoice, + ParallelToolCalls: va.ParallelToolCalls, + MaxOutputTokens: va.MaxOutputTokens, + Include: va.Include, } } // toVoiceAgent rebuilds an agent_yaml.VoiceAgent from the inline definition. func (d AgentDefinitionInline) toVoiceAgent() agent_yaml.VoiceAgent { return agent_yaml.VoiceAgent{ - AgentDefinition: d.AgentDefinition, - ModelType: d.ModelType, - Model: d.Model, - Instructions: d.Instructions, - Voice: d.Voice, - Store: d.Store, + AgentDefinition: d.AgentDefinition, + ModelType: d.ModelType, + Model: d.Model, + Instructions: d.Instructions, + Voice: d.Voice, + StructuredInputs: d.StructuredInputs, + Audio: d.Audio, + OutputModalities: d.OutputModalities, + Store: d.Store, + Tools: d.Tools, + Avatar: d.Avatar, + Greeting: d.Greeting, + Handoff: d.Handoff, + ToolChoice: d.ToolChoice, + ParallelToolCalls: d.ParallelToolCalls, + MaxOutputTokens: d.MaxOutputTokens, + Include: d.Include, } } diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go index 3318b5a554d..8b45a7637b9 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent.go @@ -130,6 +130,19 @@ func buildInvocationsWSProtocolURL(projectEndpoint, agentName string) string { ) } +func buildVoiceWSProtocolURL(projectEndpoint, agentName string) string { + projectEndpoint = strings.TrimSpace(projectEndpoint) + u, err := url.Parse(projectEndpoint) + if err != nil || u.Host == "" { + return "" + } + + return fmt.Sprintf( + "wss://%s%s/agents/%s/endpoint/protocols/voice?api-version=%s", + u.Host, strings.TrimRight(u.Path, "/"), agentName, agent_api.AgentEndpointAPIVersion, + ) +} + // ProtocolEnvSuffix pairs a user-facing label with the env var suffix // used in AGENT_{KEY}_{SUFFIX}_ENDPOINT variables. type ProtocolEnvSuffix struct { @@ -1386,13 +1399,11 @@ func (p *AgentServiceTargetProvider) Deploy( return nil, err } - // Voice agents (kind: prompt-voice) use a fundamentally different data-plane - // contract than hosted/workflow agents: a synchronous POST to /voice_agents - // that returns an AgentObject directly, with no version/polling model. Resolve - // the definition first — honoring the AGENT_DEFINITION_PATH override precedence - // so an override drives this dispatch just as it does the container path — and - // route voice to an isolated method so the container deploy path below stays - // byte-for-byte unchanged. + // Voice agents (kind: prompt-voice) use a different data-plane contract than + // hosted/workflow agents. Resolve the definition first — honoring the + // AGENT_DEFINITION_PATH override precedence so an override drives this dispatch + // just as it does the container path — and route voice to an isolated method so + // the container deploy path below stays byte-for-byte unchanged. if isVoice { return p.deployVoiceAgent(ctx, serviceConfig, voiceAgent, azdEnv, progress) } @@ -2192,12 +2203,50 @@ func (p *AgentServiceTargetProvider) deployHostedAgent( //nolint:gosec // env var key name, not a credential const voiceOverriddenHostEnvKey = "AZURE_VOICE_OVERRIDDEN_HOST" +// voiceAgentAPIEnvKey controls which voice deployment API azd uses. It defaults +// to the legacy /voice_agents path while the unified API rolls out regionally. +// Supported values: +// - legacy: POST /voice_agents with object-shaped audio.output.voice +// - unified: POST /agents or /agents/{name} with object-shaped audio.output.voice +// - unified-flat: POST /agents or /agents/{name} with flat audio.output.voice +// +//nolint:gosec // env var key name, not a credential +const voiceAgentAPIEnvKey = "AZURE_VOICE_AGENT_API" + +type voiceAgentAPIMode string + +const ( + voiceAgentAPIModeLegacy voiceAgentAPIMode = "legacy" + voiceAgentAPIModeUnified voiceAgentAPIMode = "unified" + voiceAgentAPIModeUnifiedFlat voiceAgentAPIMode = "unified-flat" +) + +func resolveVoiceAgentAPIMode(azdEnv map[string]string) (voiceAgentAPIMode, error) { + mode := strings.TrimSpace(azdEnv[voiceAgentAPIEnvKey]) + if mode == "" { + mode = strings.TrimSpace(os.Getenv(voiceAgentAPIEnvKey)) + } + mode = strings.ToLower(strings.ReplaceAll(mode, "_", "-")) + if mode == "" { + return voiceAgentAPIModeLegacy, nil + } + switch voiceAgentAPIMode(mode) { + case voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat: + return voiceAgentAPIMode(mode), nil + default: + return "", fmt.Errorf( + "%s must be one of %q, %q, or %q", + voiceAgentAPIEnvKey, voiceAgentAPIModeLegacy, voiceAgentAPIModeUnified, voiceAgentAPIModeUnifiedFlat, + ) + } +} + // deployVoiceAgent deploys a declarative (managed) voice agent (kind: -// prompt-voice) to the Foundry service. Unlike hosted agents, voice agents are -// created synchronously via a single POST to /voice_agents that returns the -// created AgentObject directly — there is no container build, no agent-version -// object, and no active-state polling. This method is intentionally isolated -// from the container deploy path so the two contracts never entangle. +// prompt-voice) to the Foundry service. The legacy /voice_agents API remains +// the default while unified /agents rolls out regionally; AZURE_VOICE_AGENT_API +// can opt into unified modes for regression and TiP validation. This method is +// intentionally isolated from the container deploy path so the two contracts +// never entangle. func (p *AgentServiceTargetProvider) deployVoiceAgent( ctx context.Context, serviceConfig *azdext.ServiceConfig, @@ -2207,7 +2256,28 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( ) (*azdext.ServiceDeployResult, error) { progress("Deploying voice agent") - request, err := agent_yaml.CreateVoiceAgentAPIRequest(va) + apiMode, err := resolveVoiceAgentAPIMode(azdEnv) + if err != nil { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + err.Error(), + fmt.Sprintf("set %s to legacy, unified, or unified-flat", voiceAgentAPIEnvKey), + ) + } + if hasAdvancedVoiceConfig(va) && apiMode != voiceAgentAPIModeUnifiedFlat { + return nil, exterrors.Validation( + exterrors.CodeInvalidParameter, + "advanced prompt-voice settings require the unified flat voice API mode", + fmt.Sprintf("set %s to unified-flat", voiceAgentAPIEnvKey), + ) + } + + var request *agent_api.CreateAgentRequest + if apiMode == voiceAgentAPIModeUnifiedFlat { + request, err = agent_yaml.CreateVoiceAgentAPIRequestFlat(va) + } else { + request, err = agent_yaml.CreateVoiceAgentAPIRequest(va) + } if err != nil { return nil, exterrors.Validation( exterrors.CodeInvalidAgentManifest, @@ -2228,24 +2298,30 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( agentClient := agent_api.NewAgentClient(projectEndpoint, p.credential) - progress("Creating voice agent") - agentObject, err := agentClient.CreateVoiceAgent( - ctx, request, agent_api.AgentEndpointAPIVersion, azdEnv[voiceOverriddenHostEnvKey], + serviceKey := p.getServiceKey(serviceConfig.Name) + agentObject, deployOp, err := p.deployVoiceAgentWithMode( + ctx, agentClient, request, apiMode, azdEnv, progress, ) if err != nil { - return nil, exterrors.ServiceFromAzure(err, exterrors.OpCreateAgent) + return nil, exterrors.ServiceFromAzure(err, deployOp) + } + if err := validateVoiceAgentDeployResponse(agentObject, apiMode); err != nil { + return nil, err } - fmt.Fprintf(os.Stderr, "Voice agent '%s' created successfully!\n", agentObject.Name) + fmt.Fprintf(os.Stderr, "Voice agent '%s' deployed successfully!\n", agentObject.Name) // Persist NAME first and ENDPOINT last. ENDPOINT is used as the voice deploy // completion marker by other commands, so avoid writing it before NAME. - serviceKey := p.getServiceKey(serviceConfig.Name) - baseEndpoint := fmt.Sprintf( - "%s/voice_agents/%s", strings.TrimRight(projectEndpoint, "/"), agentObject.Name, - ) + baseEndpoint := voiceAgentEndpoint(projectEndpoint, agentObject.Name, apiMode) + versionKey := fmt.Sprintf("AGENT_%s_VERSION", serviceKey) + versionValue := "" + if apiMode != voiceAgentAPIModeLegacy { + versionValue = agentObject.Versions.Latest.Version + } for _, envVar := range []struct{ key, value string }{ {fmt.Sprintf("AGENT_%s_NAME", serviceKey), agentObject.Name}, + {versionKey, versionValue}, {fmt.Sprintf("AGENT_%s_ENDPOINT", serviceKey), baseEndpoint}, } { if _, setErr := p.azdClient.Environment().SetValue(ctx, &azdext.SetEnvRequest{ @@ -2271,6 +2347,89 @@ func (p *AgentServiceTargetProvider) deployVoiceAgent( return &azdext.ServiceDeployResult{Artifacts: artifacts}, nil } +func hasAdvancedVoiceConfig(va agent_yaml.VoiceAgent) bool { + return len(va.StructuredInputs) > 0 || + va.Audio != nil || + len(va.OutputModalities) > 0 || + len(va.Tools) > 0 || + len(va.Avatar) > 0 || + len(va.Greeting) > 0 || + len(va.Handoff) > 0 || + va.ToolChoice != nil || + va.ParallelToolCalls != nil || + va.MaxOutputTokens != nil || + len(va.Include) > 0 +} + +func validateVoiceAgentDeployResponse(agentObject *agent_api.AgentObject, apiMode voiceAgentAPIMode) error { + if agentObject == nil { + return fmt.Errorf("malformed voice agent service response: missing agent object") + } + if strings.TrimSpace(agentObject.Name) == "" { + return fmt.Errorf("malformed voice agent service response: missing agent name") + } + if apiMode != voiceAgentAPIModeLegacy && strings.TrimSpace(agentObject.Versions.Latest.Version) == "" { + return fmt.Errorf("malformed voice agent service response: missing latest agent version") + } + return nil +} + +func (p *AgentServiceTargetProvider) deployVoiceAgentWithMode( + ctx context.Context, + agentClient *agent_api.AgentClient, + request *agent_api.CreateAgentRequest, + apiMode voiceAgentAPIMode, + azdEnv map[string]string, + progress azdext.ProgressReporter, +) (*agent_api.AgentObject, string, error) { + overriddenHost := azdEnv[voiceOverriddenHostEnvKey] + if apiMode == voiceAgentAPIModeLegacy { + progress("Creating voice agent using legacy API") + agentObject, err := agentClient.CreateVoiceAgent(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + return agentObject, exterrors.OpCreateAgent, err + } + + remoteAgent, getErr := agentClient.GetVoiceAgentUnified( + ctx, request.Name, agent_api.AgentEndpointAPIVersion, overriddenHost, + ) + shouldUpdate, decisionErr := shouldUpdateVoiceAgent(remoteAgent, getErr) + if decisionErr != nil { + return nil, exterrors.OpCreateAgent, decisionErr + } + if shouldUpdate { + progress("Updating voice agent using unified API") + updateRequest := &agent_api.UpdateAgentRequest{ + CreateAgentVersionRequest: request.CreateAgentVersionRequest, + } + agentObject, err := agentClient.UpdateVoiceAgentUnified( + ctx, request.Name, updateRequest, agent_api.AgentEndpointAPIVersion, overriddenHost, + ) + return agentObject, exterrors.OpUpdateAgent, err + } + + progress("Creating voice agent using unified API") + agentObject, err := agentClient.CreateVoiceAgentUnified(ctx, request, agent_api.AgentEndpointAPIVersion, overriddenHost) + return agentObject, exterrors.OpCreateAgent, err +} + +func shouldUpdateVoiceAgent(remoteAgent *agent_api.AgentObject, getErr error) (bool, error) { + if getErr == nil { + return remoteAgent != nil, nil + } + if respErr, ok := errors.AsType[*azcore.ResponseError](getErr); ok && respErr.StatusCode == http.StatusNotFound { + return false, nil + } + return false, getErr +} + +func voiceAgentEndpoint(projectEndpoint string, agentName string, apiMode voiceAgentAPIMode) string { + trimmedEndpoint := strings.TrimRight(projectEndpoint, "/") + if apiMode == voiceAgentAPIModeLegacy { + return fmt.Sprintf("%s/voice_agents/%s", trimmedEndpoint, agentName) + } + return buildVoiceWSProtocolURL(trimmedEndpoint, agentName) +} + // packageCodeDeploy creates a ZIP archive of the agent source code, writes it to a temp file, // and computes its SHA-256. Returns the temp file path and SHA-256 hex string. func (p *AgentServiceTargetProvider) packageCodeDeploy(ctx context.Context, serviceConfig *azdext.ServiceConfig) (string, string, error) { diff --git a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go index 68f34bb8ada..c0a62d5ea22 100644 --- a/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go +++ b/cli/azd/extensions/azure.ai.agents/internal/project/service_target_agent_test.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net" + "net/http" "os" "path/filepath" "strings" @@ -668,6 +669,120 @@ func TestAdoptServiceConfigIgnoresNilAndKeepsResolvedState(t *testing.T) { require.False(t, provider.serviceConfigResolved) } +func TestResolveVoiceAgentAPIMode_DefaultsToLegacy(t *testing.T) { + t.Setenv(voiceAgentAPIEnvKey, "") + mode, err := resolveVoiceAgentAPIMode(map[string]string{}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeLegacy, mode) +} + +func TestResolveVoiceAgentAPIMode_EnvValues(t *testing.T) { + t.Setenv(voiceAgentAPIEnvKey, "unified_flat") + mode, err := resolveVoiceAgentAPIMode(map[string]string{}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeUnifiedFlat, mode) + + mode, err = resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "unified"}) + require.NoError(t, err) + require.Equal(t, voiceAgentAPIModeUnified, mode) +} + +func TestResolveVoiceAgentAPIMode_Invalid(t *testing.T) { + _, err := resolveVoiceAgentAPIMode(map[string]string{voiceAgentAPIEnvKey: "future"}) + require.Error(t, err) + require.Contains(t, err.Error(), voiceAgentAPIEnvKey) +} + +func TestVoiceAgentEndpoint_ByMode(t *testing.T) { + projectEndpoint := "https://proj.services.ai.azure.com/api/projects/p/" + require.Equal( + t, + "https://proj.services.ai.azure.com/api/projects/p/voice_agents/my-agent", + voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeLegacy), + ) + require.Equal( + t, + "wss://proj.services.ai.azure.com/api/projects/p/agents/my-agent/endpoint/protocols/voice?api-version=v1", + voiceAgentEndpoint(projectEndpoint, "my-agent", voiceAgentAPIModeUnified), + ) +} + +func TestBuildVoiceWSProtocolURL(t *testing.T) { + got := buildVoiceWSProtocolURL( + "https://acct.services.ai.azure.com/api/projects/proj/", + "voice-agent", + ) + require.Equal( + t, + "wss://acct.services.ai.azure.com/api/projects/proj/agents/voice-agent/endpoint/protocols/voice?api-version=v1", + got, + ) +} + +func TestValidateVoiceAgentDeployResponse(t *testing.T) { + t.Run("legacy requires name only", func(t *testing.T) { + err := validateVoiceAgentDeployResponse( + &agent_api.AgentObject{Name: "voice-agent"}, + voiceAgentAPIModeLegacy, + ) + require.NoError(t, err) + }) + + t.Run("unified requires latest version", func(t *testing.T) { + agent := &agent_api.AgentObject{Name: "voice-agent"} + agent.Versions.Latest.Version = "1" + err := validateVoiceAgentDeployResponse(agent, voiceAgentAPIModeUnifiedFlat) + require.NoError(t, err) + }) + + t.Run("missing name rejected", func(t *testing.T) { + err := validateVoiceAgentDeployResponse(&agent_api.AgentObject{}, voiceAgentAPIModeLegacy) + require.ErrorContains(t, err, "missing agent name") + }) + + t.Run("unified missing version rejected", func(t *testing.T) { + err := validateVoiceAgentDeployResponse( + &agent_api.AgentObject{Name: "voice-agent"}, + voiceAgentAPIModeUnified, + ) + require.ErrorContains(t, err, "missing latest agent version") + }) +} + +func TestShouldUpdateVoiceAgent(t *testing.T) { + t.Run("remote found updates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(&agent_api.AgentObject{Name: "voice"}, nil) + require.NoError(t, err) + require.True(t, update) + }) + + t.Run("remote nil creates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, nil) + require.NoError(t, err) + require.False(t, update) + }) + + t.Run("not found creates", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, &azcore.ResponseError{StatusCode: http.StatusNotFound}) + require.NoError(t, err) + require.False(t, update) + }) + + t.Run("other get error returns error", func(t *testing.T) { + update, err := shouldUpdateVoiceAgent(nil, &azcore.ResponseError{StatusCode: http.StatusInternalServerError}) + require.Error(t, err) + require.False(t, update) + }) +} + +func TestHasAdvancedVoiceConfig(t *testing.T) { + store := false + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{})) + require.False(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Store: &store})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Audio: &agent_yaml.VoiceAudio{}})) + require.True(t, hasAdvancedVoiceConfig(agent_yaml.VoiceAgent{Tools: []map[string]any{{"type": "system"}}})) +} + func createSymlinkOrSkip(t *testing.T, oldname, newname string) { t.Helper() diff --git a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json index 553e7692fd2..28224ffb2d8 100644 --- a/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json +++ b/cli/azd/extensions/azure.ai.agents/schemas/azure.ai.agent.json @@ -1,531 +1,666 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Azure AI Agent Service Target Configuration", - "description": "Custom configuration for the Azure AI Agent Service target", - "type": "object", - "properties": { - "container": { - "$ref": "#/definitions/ContainerSettings" - }, - "deployments": { - "type": "array", - "description": "List of model deployments.", - "items": { "$ref": "#/definitions/Deployment" } - }, - "resources": { - "type": "array", - "description": "List of external resources for agent execution.", - "items": { "$ref": "#/definitions/Resource" } - }, - "toolConnections": { - "type": "array", - "description": "List of tool connections to external services (MCP tools, A2A, custom APIs) created during provisioning.", - "items": { "$ref": "#/definitions/ToolConnection" } - }, - "toolboxes": { - "type": "array", - "description": "List of toolboxes (Foundry Toolsets) to deploy.", - "items": { "$ref": "#/definitions/Toolbox" } - }, - "connections": { - "type": "array", - "description": "List of project connections to create via Bicep provisioning.", - "items": { "$ref": "#/definitions/Connection" } - }, - "memoryStores": { - "type": "array", - "description": "List of Foundry memory stores to provision (create-if-not-exists) during deployment. Memory stores let agents retain context across sessions via the memory_search tool.", - "items": { "$ref": "#/definitions/MemoryStore" } - }, - "startupCommand": { - "type": "string", - "description": "Command to start the agent server (e.g., 'python main.py'). Used by 'azd ai agent run' for local development." - }, - "activity": { - "$ref": "#/definitions/ActivitySettings" - }, - "kind": { - "type": "string", - "description": "The agent kind. 'hosted' for a containerized/code agent; 'prompt-voice' for a declarative (managed) speech-to-speech voice agent.", - "enum": ["hosted", "prompt-voice"] - }, - "modelType": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' (BYOM) references an existing Foundry model deployment.", - "enum": ["managed", "self_deployed"] - }, - "model": { - "type": "object", - "description": "Voice agent (kind: prompt-voice) speech-to-speech model (e.g. id: gpt-realtime).", - "properties": { - "id": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "Model name for managed mode (e.g. 'gpt-realtime') or existing Foundry deployment name for self_deployed mode." } - }, - "required": ["id"], - "additionalProperties": true - }, - "instructions": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." - }, - "voice": { - "type": "string", - "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." - }, - "store": { - "type": "boolean", - "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." - }, - "name": { - "type": "string", - "description": "The agent name." - }, - "displayName": { - "type": "string", - "description": "Optional human-friendly display name for the agent." - }, - "description": { - "type": "string", - "description": "Optional description of the agent." - }, - "metadata": { - "type": "object", - "description": "Optional metadata key-value pairs for the agent.", - "additionalProperties": true - }, - "protocols": { - "type": "array", - "description": "Invocation protocols the agent implements (e.g., responses, invocations, invocations_ws, a2a).", - "items": { "$ref": "#/definitions/ProtocolVersionRecord" } - }, - "agentEndpoint": { - "type": "object", - "description": "Agent endpoint configuration (protocols, version selection, auth).", - "additionalProperties": true - }, - "agentCard": { - "type": "object", - "description": "A2A discovery metadata for the agent.", - "additionalProperties": true - }, - "codeConfiguration": { - "$ref": "#/definitions/CodeConfiguration" - }, - "sessionConfiguration": { - "$ref": "#/definitions/SessionConfiguration" - }, - "policies": { - "type": "array", - "description": "Governance policies attached to the agent (e.g., Responsible AI).", - "items": { "$ref": "#/definitions/Policy" } - }, - "inputSchema": { - "type": "object", - "description": "Optional input schema for the agent.", - "additionalProperties": true - }, - "outputSchema": { - "type": "object", - "description": "Optional output schema for the agent.", - "additionalProperties": true - } - }, - "additionalProperties": true, - "allOf": [ - { - "$comment": "A prompt-voice agent must declare a speech-to-speech model; the deploy path rejects a voice service whose model.id is missing. Keep editor/schema validation aligned with that runtime requirement.", - "if": { - "properties": { - "kind": { "const": "prompt-voice" } - }, - "required": ["kind"] - }, - "then": { - "required": ["model"] - } - }, - { - "$comment": "The activity.publish block is shared publish metadata for Activity use cases (including simple). Digital Worker adds stricter requirements: publish must exist, publishScope must be tenant, and agenticUserTemplate is required to carry the persisted blueprint identity.", - "if": { - "properties": { - "activity": { - "properties": { - "useCase": { "const": "digital_worker" } - }, - "required": ["useCase"] - } - }, - "required": ["activity"] - }, - "then": { - "properties": { - "activity": { - "required": ["publish"], - "properties": { - "publish": { - "properties": { - "publishScope": { "const": "tenant" } - }, - "required": ["agenticUserTemplate"] - } - } - } - } - } - } - ], - "definitions": { - "ActivitySettings": { - "type": "object", - "description": "Activity-protocol Teams configuration. The publish block is shared metadata for Activity use cases; digital_worker applies additional constraints via allOf.", - "properties": { - "useCase": { "type": "string", "enum": ["simple", "digital_worker"] }, - "publish": { - "$ref": "#/definitions/ActivityPublishConfig", - "description": "Shared Microsoft 365 app publish metadata used by Activity use cases. For digital_worker, publishScope=tenant and agenticUserTemplate are required." - } - }, - "additionalProperties": false - }, - "ActivityPublishConfig": { - "type": "object", - "properties": { - "publishScope": { "type": "string", "enum": ["shared", "tenant"] }, - "canRespondWithoutMention": { "type": "boolean" }, - "appVersion": { "type": "string", "minLength": 1 }, - "agentDisplayName": { "type": "string", "minLength": 1 }, - "shortDescription": { "type": "string" }, - "fullDescription": { "type": "string" }, - "developerName": { "type": "string" }, - "developerWebsiteUrl": { "type": "string", "format": "uri" }, - "privacyUrl": { "type": "string", "format": "uri" }, - "termsOfUseUrl": { "type": "string", "format": "uri" }, - "agenticUserTemplate": { "$ref": "#/definitions/AgenticUserTemplateConfig" } - }, - "additionalProperties": false - }, - "AgenticUserTemplateConfig": { - "type": "object", - "properties": { - "id": { "type": "string", "minLength": 1 }, - "file": { "type": "string", "minLength": 1 }, - "schemaVersion": { "type": "string", "minLength": 1 }, - "communicationProtocol": { "type": "string", "minLength": 1 } - }, - "required": ["id", "file", "schemaVersion", "communicationProtocol"], - "additionalProperties": false - }, - "ProtocolVersionRecord": { - "type": "object", - "description": "A protocol the agent implements, with its version.", - "properties": { - "protocol": { "type": "string", "description": "Protocol name (e.g., 'responses', 'invocations', 'invocations_ws', 'a2a')." }, - "version": { "type": "string", "description": "Protocol version." } - }, - "required": ["protocol"], - "additionalProperties": false - }, - "CodeConfiguration": { - "type": "object", - "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", - "properties": { - "runtime": { "type": "string", "description": "Runtime identifier (e.g., 'python_3_12', 'dotnet_9')." }, - "entryPoint": { "type": "string", "description": "Entry point for the agent source." }, - "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } - }, - "required": ["runtime", "entryPoint"], - "additionalProperties": false - }, - "SessionConfiguration": { - "type": "object", - "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", - "properties": { - "idleTimeoutSeconds": { - "type": "integer", - "description": "Idle duration in seconds before a session's sandbox is suspended. Range 300–3600 (inclusive). Defaults to 900 when omitted.", - "minimum": 300, - "maximum": 3600 - } - }, - "additionalProperties": false - }, - "Policy": { - "type": "object", - "description": "A safety or governance policy attached to the agent.", - "properties": { - "type": { "type": "string", "description": "Policy type (e.g., 'rai_policy')." }, - "raiPolicyName": { "type": "string", "description": "ARM resource ID of the RAI policy (for type 'rai_policy')." }, - "invocationsModeration": { "$ref": "#/definitions/InvocationsModeration" } - }, - "required": ["type"], - "additionalProperties": false - }, - "InvocationsModeration": { - "type": "object", - "description": "Configures how the content-safety proxy extracts the text it submits to the RAI policy. Only supported for agents that expose the 'invocations' protocol; without it an attached RAI policy has nothing to moderate on that path.", - "properties": { - "inputContentType": { - "type": "string", - "enum": ["json", "text"], - "description": "How the request body is encoded. Defaults to 'json'." - }, - "outputContentType": { - "type": "string", - "enum": ["json", "text"], - "description": "How the response body is encoded. Defaults to 'json'." - }, - "responseMode": { - "type": "string", - "enum": ["non_streaming", "streaming", "both"], - "description": "Response shapes the agent container can produce. This declares a capability, not an input/output switch: the proxy runs exactly one output gate per response, chosen from the actual response Content-Type." - }, - "inputPaths": { - "type": "array", - "description": "JSONPath expressions selecting request text. Required when inputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "type": "string" } - }, - "outputPaths": { - "type": "array", - "description": "JSONPath expressions selecting buffered response text. Required when responseMode includes non-streaming and outputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "type": "string" } - }, - "streamSelectors": { - "type": "array", - "description": "Locates text within server-sent event frames. Required when responseMode includes streaming and outputContentType is 'json' or omitted.", - "minItems": 1, - "items": { "$ref": "#/definitions/SseTextSelector" } - } - }, - "required": ["responseMode"], - "allOf": [ - { - "if": { "$ref": "#/definitions/InvocationsInputIsJson" }, - "then": { "required": ["inputPaths"] } - }, - { - "if": { - "allOf": [ - { "properties": { "responseMode": { "enum": ["non_streaming", "both"] } }, "required": ["responseMode"] }, - { "$ref": "#/definitions/InvocationsOutputIsJson" } - ] - }, - "then": { "required": ["outputPaths"] } - }, - { - "if": { - "allOf": [ - { "properties": { "responseMode": { "enum": ["streaming", "both"] } }, "required": ["responseMode"] }, - { "$ref": "#/definitions/InvocationsOutputIsJson" } - ] - }, - "then": { "required": ["streamSelectors"] } - } - ], - "additionalProperties": false - }, - "InvocationsInputIsJson": { - "description": "Matches when inputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", - "anyOf": [ - { "properties": { "inputContentType": { "const": "json" } }, "required": ["inputContentType"] }, - { "not": { "required": ["inputContentType"] } } - ] - }, - "InvocationsOutputIsJson": { - "description": "Matches when outputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", - "anyOf": [ - { "properties": { "outputContentType": { "const": "json" } }, "required": ["outputContentType"] }, - { "not": { "required": ["outputContentType"] } } - ] - }, - "SseTextSelector": { - "type": "object", - "description": "Locates the text to moderate inside a single server-sent event frame.", - "properties": { - "eventType": { "type": "string", "pattern": "\\S", "description": "SSE event name this selector applies to." }, - "textField": { "type": "string", "description": "JSONPath expression, relative to the frame payload, holding the text." } - }, - "required": ["eventType"], - "additionalProperties": false - }, - "ContainerSettings": { - "type": "object", - "description": "Container configuration for the Azure AI Agent Service target", - "properties": { - "resources": { - "$ref": "#/definitions/ResourceSettings" - } - }, - "additionalProperties": false - }, - "ResourceSettings": { - "type": "object", - "description": "Resource configuration for the Azure AI Agent Service target", - "properties": { - "memory": { - "type": "string", - "description": "Memory allocation (e.g., '1Gi', '512Mi')", - "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$" - }, - "cpu": { - "type": "string", - "description": "CPU allocation (e.g., '1', '500m')", - "pattern": "^[0-9]+(\\.[0-9]+)?m?$" - } - }, - "additionalProperties": false - }, - "Deployment": { - "type": "object", - "description": "A single model deployment.", - "properties": { - "name": { "type": "string", "description": "Name of the model deployment." }, - "model": { "$ref": "#/definitions/DeploymentModel" }, - "sku": { "$ref": "#/definitions/DeploymentSku" } - }, - "required": ["name", "model", "sku"], - "additionalProperties": false - }, - "DeploymentModel": { - "type": "object", - "description": "Model configuration for a model deployment.", - "properties": { - "name": { "type": "string", "description": "Model name." }, - "format": { "type": "string", "description": "Model format." }, - "version": { "type": "string", "description": "Model version." } - }, - "required": ["name", "format", "version"], - "additionalProperties": false - }, - "DeploymentSku": { - "type": "object", - "description": "SKU configuration for a deployment.", - "properties": { - "name": { "type": "string", "description": "SKU name." }, - "capacity": { "type": "integer", "description": "SKU capacity." } - }, - "required": ["name", "capacity"], - "additionalProperties": false - }, - "Resource": { - "type": "object", - "description": "External resource for agent execution.", - "properties": { - "resource": { "type": "string", "description": "Resource identifier." }, - "connectionName": { "type": "string", "description": "Connection name for the resource." } - }, - "required": ["resource", "connectionName"], - "additionalProperties": false - }, - "ToolConnection": { - "type": "object", - "description": "A connection to an external service (MCP tool, A2A, custom API) created via Bicep during provisioning.", - "properties": { - "name": { "type": "string", "description": "Connection name used as project_connection_id in toolbox tools." }, - "category": { "type": "string", "description": "Connection category (e.g., 'RemoteTool')." }, - "target": { "type": "string", "description": "Target endpoint URL for the connection." }, - "authType": { - "type": "string", - "description": "Authentication type for the connection.", - "enum": ["AAD", "AccessKey", "AccountKey", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "ServicePrincipal", "UsernamePassword", "ProjectManagedIdentity", "UserEntraToken", "AgenticIdentityToken"] - }, - "credentials": { - "type": "object", - "description": "Credentials for the connection. Values may contain ${ENV_VAR} references resolved at provision time." - }, - "metadata": { - "type": "object", - "description": "Additional metadata for the connection.", - "additionalProperties": { "type": "string" } - } - }, - "required": ["name", "category", "target", "authType"], - "additionalProperties": false - }, - "Toolbox": { - "type": "object", - "description": "A reusable collection of tools deployed as a Foundry Toolset.", - "properties": { - "name": { "type": "string", "description": "Name of the toolbox." }, - "description": { "type": "string", "description": "Description of the toolbox." }, - "tools": { - "type": "array", - "description": "List of tools in the toolbox. Each tool is an object with properties passed to the Foundry Toolsets API.", - "items": { "type": "object" } - } - }, - "required": ["name", "tools"], - "additionalProperties": false - }, - "Connection": { - "type": "object", - "description": "A project connection matching the Bicep ConnectionPropertiesV2 spec.", - "properties": { - "name": { "type": "string", "description": "Connection name.", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]{2,32}$" }, - "category": { "type": "string", "description": "Connection category (e.g., 'CustomKeys', 'AzureOpenAI', 'CognitiveSearch', 'RemoteTool')." }, - "target": { "type": "string", "description": "Target endpoint URL for the connection." }, - "authType": { - "type": "string", - "description": "Authentication type.", - "enum": ["AAD", "AccessKey", "AccountKey", "AgenticIdentity", "AgenticIdentityToken", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "SAS", "ServicePrincipal", "UsernamePassword", "UserEntraToken", "ProjectManagedIdentity"] - }, - "credentials": { - "type": "object", - "description": "Authentication credentials. Structure depends on authType." - }, - "metadata": { - "type": "object", - "description": "Additional metadata as key-value pairs.", - "additionalProperties": { "type": "string" } - }, - "authorizationUrl": { "type": "string", "description": "OAuth2 authorization endpoint URL (required for OAuth2 authType)." }, - "tokenUrl": { "type": "string", "description": "OAuth2 token endpoint URL (required for OAuth2 authType)." }, - "refreshUrl": { "type": "string", "description": "OAuth2 token refresh URL (optional for OAuth2 authType)." }, - "scopes": { - "type": "array", - "description": "OAuth2 scopes to request (optional for OAuth2 authType).", - "items": { "type": "string" } - }, - "audience": { "type": "string", "description": "Token audience for AAD/ProjectManagedIdentity/AgenticIdentity/AgenticIdentityToken/UserEntraToken auth types." }, - "connectorName": { "type": "string", "description": "Connector name for Oauth2 auth type." }, - "expiryTime": { "type": "string", "description": "Connection expiry time." }, - "isSharedToAll": { "type": "boolean", "description": "Whether the connection is shared to all users." }, - "sharedUserList": { - "type": "array", - "description": "List of users the connection is shared with.", - "items": { "type": "string" } - }, - "peRequirement": { "type": "string", "description": "Private endpoint requirement." }, - "peStatus": { "type": "string", "description": "Private endpoint status." }, - "useWorkspaceManagedIdentity": { "type": "boolean", "description": "Whether to use workspace managed identity." }, - "error": { "type": "string", "description": "Error information." } - }, - "required": ["name", "category", "target", "authType"], - "additionalProperties": false - }, - "MemoryStore": { - "type": "object", - "description": "A Foundry memory store provisioned (create-if-not-exists) during deployment. Backs the agent's memory_search tool so the agent can retain context across sessions.", - "properties": { - "name": { "type": "string", "description": "Name of the memory store." }, - "description": { "type": "string", "description": "Description of the memory store." }, - "chatModel": { "type": "string", "description": "Chat model deployment name used by the memory store (must exist in the Foundry project)." }, - "embeddingModel": { "type": "string", "description": "Embedding model deployment name used by the memory store (must exist in the Foundry project)." }, - "options": { - "type": "object", - "description": "Optional extraction and retention settings for the memory store.", - "properties": { - "chatSummaryEnabled": { "type": "boolean", "description": "Enable rolling chat-summary memory." }, - "userProfileEnabled": { "type": "boolean", "description": "Enable durable user-profile memory." }, - "proceduralMemoryEnabled": { "type": "boolean", "description": "Enable procedural (how-to) memory." }, - "defaultTtlSeconds": { "type": "integer", "description": "Default time-to-live (seconds) for new memory entries. 0 means no expiration." }, - "userProfileDetails": { "type": "string", "description": "Guidance on what user-profile information the agent should retain or avoid." } - }, - "additionalProperties": false - } - }, - "required": ["name", "chatModel", "embeddingModel"], - "additionalProperties": false - } - } -} +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Azure AI Agent Service Target Configuration", + "description": "Custom configuration for the Azure AI Agent Service target", + "type": "object", + "properties": { + "container": { + "$ref": "#/definitions/ContainerSettings" + }, + "deployments": { + "type": "array", + "description": "List of model deployments.", + "items": { "$ref": "#/definitions/Deployment" } + }, + "resources": { + "type": "array", + "description": "List of external resources for agent execution.", + "items": { "$ref": "#/definitions/Resource" } + }, + "toolConnections": { + "type": "array", + "description": "List of tool connections to external services (MCP tools, A2A, custom APIs) created during provisioning.", + "items": { "$ref": "#/definitions/ToolConnection" } + }, + "toolboxes": { + "type": "array", + "description": "List of toolboxes (Foundry Toolsets) to deploy.", + "items": { "$ref": "#/definitions/Toolbox" } + }, + "connections": { + "type": "array", + "description": "List of project connections to create via Bicep provisioning.", + "items": { "$ref": "#/definitions/Connection" } + }, + "memoryStores": { + "type": "array", + "description": "List of Foundry memory stores to provision (create-if-not-exists) during deployment. Memory stores let agents retain context across sessions via the memory_search tool.", + "items": { "$ref": "#/definitions/MemoryStore" } + }, + "startupCommand": { + "type": "string", + "description": "Command to start the agent server (e.g., 'python main.py'). Used by 'azd ai agent run' for local development." + }, + "activity": { + "$ref": "#/definitions/ActivitySettings" + }, + "kind": { + "type": "string", + "description": "The agent kind. 'hosted' for a containerized/code agent; 'prompt-voice' for a declarative (managed) speech-to-speech voice agent.", + "enum": ["hosted", "prompt-voice"] + }, + "modelType": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) model-inference mode. 'managed' uses a Voice Live-hosted model; 'self_deployed' (BYOM) references an existing Foundry model deployment.", + "enum": ["managed", "self_deployed"] + }, + "model": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) speech-to-speech model (e.g. id: gpt-realtime).", + "properties": { + "id": { "type": "string", "minLength": 1, "pattern": "\\S", "description": "Model name for managed mode (e.g. 'gpt-realtime') or existing Foundry deployment name for self_deployed mode." } + }, + "required": ["id"], + "additionalProperties": true + }, + "instructions": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) system prompt for the assistant." + }, + "voice": { + "type": "string", + "description": "Voice agent (kind: prompt-voice) output voice name (e.g. 'en-US-Ava:DragonHDLatestNeural' for an Azure Neural voice, or 'alloy' for an OpenAI realtime voice)." + }, + "structuredInputs": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) structured prompt inputs. Use description, defaultValue, schema, and required; azd maps defaultValue to the service wire field default_value.", + "additionalProperties": true + }, + "audio": { + "$ref": "#/definitions/VoiceAudio" + }, + "outputModalities": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) output modalities. Well-known values are audio, text, animation, and avatar.", + "items": { "type": "string", "minLength": 1 } + }, + "store": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) server-side logging toggle (transcript + per-turn audio). Defaults to false when omitted." + }, + "tools": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) tools. Direct tool types include function, mcp, system, and toolbox.", + "items": { "type": "object", "additionalProperties": true } + }, + "avatar": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) avatar configuration.", + "additionalProperties": true + }, + "greeting": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) greeting configuration, such as template or llm_generated.", + "additionalProperties": true + }, + "handoff": { + "type": "object", + "description": "Voice agent (kind: prompt-voice) handoff configuration.", + "additionalProperties": true + }, + "toolChoice": { + "description": "Voice agent (kind: prompt-voice) tool choice behavior, such as none, auto, required, or a tool choice object." + }, + "parallelToolCalls": { + "type": "boolean", + "description": "Voice agent (kind: prompt-voice) parallel tool call toggle." + }, + "maxOutputTokens": { + "description": "Voice agent (kind: prompt-voice) maximum output tokens. Use an integer or a service-supported string such as inf." + }, + "include": { + "type": "array", + "description": "Voice agent (kind: prompt-voice) extra service response fields to include.", + "items": { "type": "string" } + }, + "name": { + "type": "string", + "description": "The agent name." + }, + "displayName": { + "type": "string", + "description": "Optional human-friendly display name for the agent." + }, + "description": { + "type": "string", + "description": "Optional description of the agent." + }, + "metadata": { + "type": "object", + "description": "Optional metadata key-value pairs for the agent.", + "additionalProperties": true + }, + "protocols": { + "type": "array", + "description": "Invocation protocols the agent implements (e.g., responses, invocations, invocations_ws, a2a).", + "items": { "$ref": "#/definitions/ProtocolVersionRecord" } + }, + "agentEndpoint": { + "type": "object", + "description": "Agent endpoint configuration (protocols, version selection, auth).", + "additionalProperties": true + }, + "agentCard": { + "type": "object", + "description": "A2A discovery metadata for the agent.", + "additionalProperties": true + }, + "codeConfiguration": { + "$ref": "#/definitions/CodeConfiguration" + }, + "sessionConfiguration": { + "$ref": "#/definitions/SessionConfiguration" + }, + "policies": { + "type": "array", + "description": "Governance policies attached to the agent (e.g., Responsible AI).", + "items": { "$ref": "#/definitions/Policy" } + }, + "inputSchema": { + "type": "object", + "description": "Optional input schema for the agent.", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "description": "Optional output schema for the agent.", + "additionalProperties": true + } + }, + "additionalProperties": true, + "allOf": [ + { + "$comment": "A prompt-voice agent must declare a speech-to-speech model; the deploy path rejects a voice service whose model.id is missing. Keep editor/schema validation aligned with that runtime requirement.", + "if": { + "properties": { + "kind": { "const": "prompt-voice" } + }, + "required": ["kind"] + }, + "then": { + "required": ["model"] + } + }, + { + "$comment": "The activity.publish block is shared publish metadata for Activity use cases (including simple). Digital Worker adds stricter requirements: publish must exist, publishScope must be tenant, and agenticUserTemplate is required to carry the persisted blueprint identity.", + "if": { + "properties": { + "activity": { + "properties": { + "useCase": { "const": "digital_worker" } + }, + "required": ["useCase"] + } + }, + "required": ["activity"] + }, + "then": { + "properties": { + "activity": { + "required": ["publish"], + "properties": { + "publish": { + "properties": { + "publishScope": { "const": "tenant" } + }, + "required": ["agenticUserTemplate"] + } + } + } + } + } + } + ], + "definitions": { + "ActivitySettings": { + "type": "object", + "description": "Activity-protocol Teams configuration. The publish block is shared metadata for Activity use cases; digital_worker applies additional constraints via allOf.", + "properties": { + "useCase": { "type": "string", "enum": ["simple", "digital_worker"] }, + "publish": { + "$ref": "#/definitions/ActivityPublishConfig", + "description": "Shared Microsoft 365 app publish metadata used by Activity use cases. For digital_worker, publishScope=tenant and agenticUserTemplate are required." + } + }, + "additionalProperties": false + }, + "ActivityPublishConfig": { + "type": "object", + "properties": { + "publishScope": { "type": "string", "enum": ["shared", "tenant"] }, + "canRespondWithoutMention": { "type": "boolean" }, + "appVersion": { "type": "string", "minLength": 1 }, + "agentDisplayName": { "type": "string", "minLength": 1 }, + "shortDescription": { "type": "string" }, + "fullDescription": { "type": "string" }, + "developerName": { "type": "string" }, + "developerWebsiteUrl": { "type": "string", "format": "uri" }, + "privacyUrl": { "type": "string", "format": "uri" }, + "termsOfUseUrl": { "type": "string", "format": "uri" }, + "agenticUserTemplate": { "$ref": "#/definitions/AgenticUserTemplateConfig" } + }, + "additionalProperties": false + }, + "AgenticUserTemplateConfig": { + "type": "object", + "properties": { + "id": { "type": "string", "minLength": 1 }, + "file": { "type": "string", "minLength": 1 }, + "schemaVersion": { "type": "string", "minLength": 1 }, + "communicationProtocol": { "type": "string", "minLength": 1 } + }, + "required": ["id", "file", "schemaVersion", "communicationProtocol"], + "additionalProperties": false + }, + "ProtocolVersionRecord": { + "type": "object", + "description": "A protocol the agent implements, with its version.", + "properties": { + "protocol": { "type": "string", "description": "Protocol name (e.g., 'responses', 'invocations', 'invocations_ws', 'a2a')." }, + "version": { "type": "string", "description": "Protocol version." } + }, + "required": ["protocol"], + "additionalProperties": false + }, + "CodeConfiguration": { + "type": "object", + "description": "Code deploy configuration. When present, the agent is deployed from source (ZIP) instead of a container image.", + "properties": { + "runtime": { "type": "string", "description": "Runtime identifier (e.g., 'python_3_12', 'dotnet_9')." }, + "entryPoint": { "type": "string", "description": "Entry point for the agent source." }, + "dependencyResolution": { "type": "string", "description": "Dependency resolution mode (e.g., 'bundled', 'remote_build')." } + }, + "required": ["runtime", "entryPoint"], + "additionalProperties": false + }, + "VoiceAudio": { + "type": "object", + "description": "Prompt voice input and output audio configuration. Requires AZURE_VOICE_AGENT_API=unified-flat for deployment.", + "properties": { + "input": { "$ref": "#/definitions/VoiceAudioInput" }, + "output": { "$ref": "#/definitions/VoiceAudioOutput" } + }, + "additionalProperties": false + }, + "VoiceAudioInput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "noiseReduction": { "$ref": "#/definitions/VoiceNoiseReduction" }, + "echoCancellation": { "type": "object", "additionalProperties": true }, + "turnDetection": { "$ref": "#/definitions/VoiceTurnDetection" }, + "transcription": { "$ref": "#/definitions/VoiceTranscription" } + }, + "additionalProperties": false + }, + "VoiceAudioOutput": { + "type": "object", + "properties": { + "format": { "$ref": "#/definitions/VoiceAudioFormat" }, + "voice": { "$ref": "#/definitions/VoiceConfig" }, + "speed": { "type": "number", "minimum": 0.25, "maximum": 1.5 } + }, + "additionalProperties": false + }, + "VoiceAudioFormat": { + "type": "object", + "properties": { + "type": { "type": "string", "enum": ["audio/pcm", "audio/pcmu", "audio/pcma"] }, + "rate": { "type": "integer", "minimum": 1 } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceNoiseReduction": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include near_field, far_field, and azure_deep_noise_suppression." } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTurnDetection": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Well-known values include server_vad, semantic_vad, and azure_semantic_vad." }, + "threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "prefixPaddingMs": { "type": "integer", "minimum": 0 }, + "silenceDurationMs": { "type": "integer", "minimum": 0 }, + "createResponse": { "type": "boolean" }, + "eagerness": { "type": "string" }, + "speechDurationMs": { "type": "integer", "minimum": 0 }, + "removeFillerWords": { "type": "boolean" }, + "interruptResponse": { "type": "boolean" }, + "languages": { "type": "array", "items": { "type": "string" } }, + "autoTruncate": { "type": "boolean" } + }, + "required": ["type"], + "additionalProperties": false + }, + "VoiceTranscription": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "language": { "type": "string" }, + "prompt": { "type": "string" } + }, + "additionalProperties": false + }, + "VoiceConfig": { + "type": "object", + "properties": { + "type": { "type": "string", "description": "Voice provider type, such as openai, azure_standard, or azure-standard." }, + "name": { "type": "string" }, + "style": { "type": "string" }, + "pitch": { "type": "string" }, + "rate": { "type": "string" }, + "locale": { "type": "string" }, + "volume": { "type": "string" } + }, + "required": ["type", "name"], + "additionalProperties": false + }, + "SessionConfiguration": { + "type": "object", + "description": "Optional hosted-agent session runtime settings. When omitted, the service applies its defaults (idle timeout 900 seconds).", + "properties": { + "idleTimeoutSeconds": { + "type": "integer", + "description": "Idle duration in seconds before a session's sandbox is suspended. Range 300–3600 (inclusive). Defaults to 900 when omitted.", + "minimum": 300, + "maximum": 3600 + } + }, + "additionalProperties": false + }, + "Policy": { + "type": "object", + "description": "A safety or governance policy attached to the agent.", + "properties": { + "type": { "type": "string", "description": "Policy type (e.g., 'rai_policy')." }, + "raiPolicyName": { "type": "string", "description": "ARM resource ID of the RAI policy (for type 'rai_policy')." }, + "invocationsModeration": { "$ref": "#/definitions/InvocationsModeration" } + }, + "required": ["type"], + "additionalProperties": false + }, + "InvocationsModeration": { + "type": "object", + "description": "Configures how the content-safety proxy extracts the text it submits to the RAI policy. Only supported for agents that expose the 'invocations' protocol; without it an attached RAI policy has nothing to moderate on that path.", + "properties": { + "inputContentType": { + "type": "string", + "enum": ["json", "text"], + "description": "How the request body is encoded. Defaults to 'json'." + }, + "outputContentType": { + "type": "string", + "enum": ["json", "text"], + "description": "How the response body is encoded. Defaults to 'json'." + }, + "responseMode": { + "type": "string", + "enum": ["non_streaming", "streaming", "both"], + "description": "Response shapes the agent container can produce. This declares a capability, not an input/output switch: the proxy runs exactly one output gate per response, chosen from the actual response Content-Type." + }, + "inputPaths": { + "type": "array", + "description": "JSONPath expressions selecting request text. Required when inputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "type": "string" } + }, + "outputPaths": { + "type": "array", + "description": "JSONPath expressions selecting buffered response text. Required when responseMode includes non-streaming and outputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "type": "string" } + }, + "streamSelectors": { + "type": "array", + "description": "Locates text within server-sent event frames. Required when responseMode includes streaming and outputContentType is 'json' or omitted.", + "minItems": 1, + "items": { "$ref": "#/definitions/SseTextSelector" } + } + }, + "required": ["responseMode"], + "allOf": [ + { + "if": { "$ref": "#/definitions/InvocationsInputIsJson" }, + "then": { "required": ["inputPaths"] } + }, + { + "if": { + "allOf": [ + { "properties": { "responseMode": { "enum": ["non_streaming", "both"] } }, "required": ["responseMode"] }, + { "$ref": "#/definitions/InvocationsOutputIsJson" } + ] + }, + "then": { "required": ["outputPaths"] } + }, + { + "if": { + "allOf": [ + { "properties": { "responseMode": { "enum": ["streaming", "both"] } }, "required": ["responseMode"] }, + { "$ref": "#/definitions/InvocationsOutputIsJson" } + ] + }, + "then": { "required": ["streamSelectors"] } + } + ], + "additionalProperties": false + }, + "InvocationsInputIsJson": { + "description": "Matches when inputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", + "anyOf": [ + { "properties": { "inputContentType": { "const": "json" } }, "required": ["inputContentType"] }, + { "not": { "required": ["inputContentType"] } } + ] + }, + "InvocationsOutputIsJson": { + "description": "Matches when outputContentType resolves to 'json', i.e. it is set to 'json' or omitted entirely (the default).", + "anyOf": [ + { "properties": { "outputContentType": { "const": "json" } }, "required": ["outputContentType"] }, + { "not": { "required": ["outputContentType"] } } + ] + }, + "SseTextSelector": { + "type": "object", + "description": "Locates the text to moderate inside a single server-sent event frame.", + "properties": { + "eventType": { "type": "string", "pattern": "\\S", "description": "SSE event name this selector applies to." }, + "textField": { "type": "string", "description": "JSONPath expression, relative to the frame payload, holding the text." } + }, + "required": ["eventType"], + "additionalProperties": false + }, + "ContainerSettings": { + "type": "object", + "description": "Container configuration for the Azure AI Agent Service target", + "properties": { + "resources": { + "$ref": "#/definitions/ResourceSettings" + } + }, + "additionalProperties": false + }, + "ResourceSettings": { + "type": "object", + "description": "Resource configuration for the Azure AI Agent Service target", + "properties": { + "memory": { + "type": "string", + "description": "Memory allocation (e.g., '1Gi', '512Mi')", + "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$" + }, + "cpu": { + "type": "string", + "description": "CPU allocation (e.g., '1', '500m')", + "pattern": "^[0-9]+(\\.[0-9]+)?m?$" + } + }, + "additionalProperties": false + }, + "Deployment": { + "type": "object", + "description": "A single model deployment.", + "properties": { + "name": { "type": "string", "description": "Name of the model deployment." }, + "model": { "$ref": "#/definitions/DeploymentModel" }, + "sku": { "$ref": "#/definitions/DeploymentSku" } + }, + "required": ["name", "model", "sku"], + "additionalProperties": false + }, + "DeploymentModel": { + "type": "object", + "description": "Model configuration for a model deployment.", + "properties": { + "name": { "type": "string", "description": "Model name." }, + "format": { "type": "string", "description": "Model format." }, + "version": { "type": "string", "description": "Model version." } + }, + "required": ["name", "format", "version"], + "additionalProperties": false + }, + "DeploymentSku": { + "type": "object", + "description": "SKU configuration for a deployment.", + "properties": { + "name": { "type": "string", "description": "SKU name." }, + "capacity": { "type": "integer", "description": "SKU capacity." } + }, + "required": ["name", "capacity"], + "additionalProperties": false + }, + "Resource": { + "type": "object", + "description": "External resource for agent execution.", + "properties": { + "resource": { "type": "string", "description": "Resource identifier." }, + "connectionName": { "type": "string", "description": "Connection name for the resource." } + }, + "required": ["resource", "connectionName"], + "additionalProperties": false + }, + "ToolConnection": { + "type": "object", + "description": "A connection to an external service (MCP tool, A2A, custom API) created via Bicep during provisioning.", + "properties": { + "name": { "type": "string", "description": "Connection name used as project_connection_id in toolbox tools." }, + "category": { "type": "string", "description": "Connection category (e.g., 'RemoteTool')." }, + "target": { "type": "string", "description": "Target endpoint URL for the connection." }, + "authType": { + "type": "string", + "description": "Authentication type for the connection.", + "enum": ["AAD", "AccessKey", "AccountKey", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "ServicePrincipal", "UsernamePassword", "ProjectManagedIdentity", "UserEntraToken", "AgenticIdentityToken"] + }, + "credentials": { + "type": "object", + "description": "Credentials for the connection. Values may contain ${ENV_VAR} references resolved at provision time." + }, + "metadata": { + "type": "object", + "description": "Additional metadata for the connection.", + "additionalProperties": { "type": "string" } + } + }, + "required": ["name", "category", "target", "authType"], + "additionalProperties": false + }, + "Toolbox": { + "type": "object", + "description": "A reusable collection of tools deployed as a Foundry Toolset.", + "properties": { + "name": { "type": "string", "description": "Name of the toolbox." }, + "description": { "type": "string", "description": "Description of the toolbox." }, + "tools": { + "type": "array", + "description": "List of tools in the toolbox. Each tool is an object with properties passed to the Foundry Toolsets API.", + "items": { "type": "object" } + } + }, + "required": ["name", "tools"], + "additionalProperties": false + }, + "Connection": { + "type": "object", + "description": "A project connection matching the Bicep ConnectionPropertiesV2 spec.", + "properties": { + "name": { "type": "string", "description": "Connection name.", "pattern": "^[a-zA-Z0-9][a-zA-Z0-9_-]{2,32}$" }, + "category": { "type": "string", "description": "Connection category (e.g., 'CustomKeys', 'AzureOpenAI', 'CognitiveSearch', 'RemoteTool')." }, + "target": { "type": "string", "description": "Target endpoint URL for the connection." }, + "authType": { + "type": "string", + "description": "Authentication type.", + "enum": ["AAD", "AccessKey", "AccountKey", "AgenticIdentity", "AgenticIdentityToken", "ApiKey", "CustomKeys", "ManagedIdentity", "None", "OAuth2", "PAT", "SAS", "ServicePrincipal", "UsernamePassword", "UserEntraToken", "ProjectManagedIdentity"] + }, + "credentials": { + "type": "object", + "description": "Authentication credentials. Structure depends on authType." + }, + "metadata": { + "type": "object", + "description": "Additional metadata as key-value pairs.", + "additionalProperties": { "type": "string" } + }, + "authorizationUrl": { "type": "string", "description": "OAuth2 authorization endpoint URL (required for OAuth2 authType)." }, + "tokenUrl": { "type": "string", "description": "OAuth2 token endpoint URL (required for OAuth2 authType)." }, + "refreshUrl": { "type": "string", "description": "OAuth2 token refresh URL (optional for OAuth2 authType)." }, + "scopes": { + "type": "array", + "description": "OAuth2 scopes to request (optional for OAuth2 authType).", + "items": { "type": "string" } + }, + "audience": { "type": "string", "description": "Token audience for AAD/ProjectManagedIdentity/AgenticIdentity/AgenticIdentityToken/UserEntraToken auth types." }, + "connectorName": { "type": "string", "description": "Connector name for Oauth2 auth type." }, + "expiryTime": { "type": "string", "description": "Connection expiry time." }, + "isSharedToAll": { "type": "boolean", "description": "Whether the connection is shared to all users." }, + "sharedUserList": { + "type": "array", + "description": "List of users the connection is shared with.", + "items": { "type": "string" } + }, + "peRequirement": { "type": "string", "description": "Private endpoint requirement." }, + "peStatus": { "type": "string", "description": "Private endpoint status." }, + "useWorkspaceManagedIdentity": { "type": "boolean", "description": "Whether to use workspace managed identity." }, + "error": { "type": "string", "description": "Error information." } + }, + "required": ["name", "category", "target", "authType"], + "additionalProperties": false + }, + "MemoryStore": { + "type": "object", + "description": "A Foundry memory store provisioned (create-if-not-exists) during deployment. Backs the agent's memory_search tool so the agent can retain context across sessions.", + "properties": { + "name": { "type": "string", "description": "Name of the memory store." }, + "description": { "type": "string", "description": "Description of the memory store." }, + "chatModel": { "type": "string", "description": "Chat model deployment name used by the memory store (must exist in the Foundry project)." }, + "embeddingModel": { "type": "string", "description": "Embedding model deployment name used by the memory store (must exist in the Foundry project)." }, + "options": { + "type": "object", + "description": "Optional extraction and retention settings for the memory store.", + "properties": { + "chatSummaryEnabled": { "type": "boolean", "description": "Enable rolling chat-summary memory." }, + "userProfileEnabled": { "type": "boolean", "description": "Enable durable user-profile memory." }, + "proceduralMemoryEnabled": { "type": "boolean", "description": "Enable procedural (how-to) memory." }, + "defaultTtlSeconds": { "type": "integer", "description": "Default time-to-live (seconds) for new memory entries. 0 means no expiration." }, + "userProfileDetails": { "type": "string", "description": "Guidance on what user-profile information the agent should retain or avoid." } + }, + "additionalProperties": false + } + }, + "required": ["name", "chatModel", "embeddingModel"], + "additionalProperties": false + } + } +}