Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cli/azd/extensions/azure.ai.agents/cspell.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ words:
# Voice (prompt-voice) agents
- BYOM
- Nanami
- pcma
- pcmu
- webrtc
# Azure region names
- australiaeast
- brazilsouth
Expand Down
27 changes: 13 additions & 14 deletions cli/azd/extensions/azure.ai.agents/internal/cmd/nextstep/state.go
Original file line number Diff line number Diff line change
Expand Up @@ -822,23 +822,22 @@ func isDeployed(
*errs = append(*errs, fmt.Errorf("read %s: %w", key, err))
return false
}
if value != "" {
return true
}

// Voice agents (kind: prompt-voice) deploy without an agent-version object,
// so they never set AGENT_<KEY>_VERSION. Fall back to the base endpoint
// marker, which every voice deploy writes, so a successfully created voice
// agent is not reported as undeployed. Gate this on the service's actual
// declared kind: a hosted agent whose deploy partially failed can also
// present an empty VERSION with a lingering ENDPOINT, and must stay reported
// as not-deployed. This mirrors the kind gate in
if !isVoice {
return value != ""
}

// Voice deploys use the base ENDPOINT env var as the completion marker. The
// legacy voice API does not produce AGENT_<KEY>_VERSION, and unified voice
// deploys write VERSION before ENDPOINT to keep ENDPOINT as the final marker.
// Require ENDPOINT for voice even when VERSION is present, otherwise a partial
// env write could be reported as deployed before the callable endpoint was
// persisted. Gate this on the service's actual declared kind: a hosted agent
// whose deploy partially failed can also present an empty VERSION with a
// lingering ENDPOINT, and must stay reported as not-deployed. This mirrors the
// kind gate in
// AgentServiceTargetProvider.Endpoints (project package); the two live in
// separate packages because project imports nextstep, so a literally shared
// helper would create an import cycle.
if !isVoice {
return false
}
endpointKey := fmt.Sprintf(agentEndpointVarFormat, serviceKey(serviceName))
endpointValue, err := src.EnvValue(ctx, envName, endpointKey)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -739,10 +739,10 @@ func TestServiceKey(t *testing.T) {
}
}

// TestIsDeployed_VoiceEndpointFallback verifies that a voice agent — which sets
// only AGENT_<KEY>_NAME and AGENT_<KEY>_ENDPOINT, never AGENT_<KEY>_VERSION — is
// still reported as deployed via the base endpoint marker, while an agent with
// neither version nor endpoint is reported undeployed.
// TestIsDeployed_VoiceEndpointFallback verifies that voice readiness is based
// on the base endpoint marker. Legacy voice agents never set VERSION, while
// unified voice agents set VERSION before ENDPOINT; in both cases ENDPOINT is
// the deploy completion marker.
func TestIsDeployed_VoiceEndpointFallback(t *testing.T) {
t.Parallel()

Expand All @@ -757,6 +757,21 @@ func TestIsDeployed_VoiceEndpointFallback(t *testing.T) {
values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"},
want: true,
},
{
name: "version set but endpoint missing: undeployed (voice agent partial write)",
values: map[string]string{"env1/AGENT_VOICE_SVC_VERSION": "1"},
isVoice: true,
want: false,
},
{
name: "version and endpoint set: deployed (unified voice agent)",
values: map[string]string{
"env1/AGENT_VOICE_SVC_VERSION": "1",
"env1/AGENT_VOICE_SVC_ENDPOINT": "wss://x/agents/a/endpoint/protocols/voice?api-version=v1",
},
isVoice: true,
want: true,
},
{
name: "no version but base endpoint set: deployed (voice agent)",
values: map[string]string{"env1/AGENT_VOICE_SVC_ENDPOINT": "https://x/voice_agents/a"},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ const (
// VoiceAudioFormat describes a PCM audio stream format (e.g. audio/pcm @ 24 kHz).
type VoiceAudioFormat struct {
Type string `json:"type"`
Rate int `json:"rate"`
Rate *int `json:"rate,omitempty"`
}

// VoiceTurnDetection configures server-side voice-activity detection so the
Expand All @@ -370,32 +370,69 @@ type VoiceTurnDetection struct {
Threshold *float64 `json:"threshold,omitempty"`
PrefixPaddingMs *int `json:"prefix_padding_ms,omitempty"`
SilenceDurationMs *int `json:"silence_duration_ms,omitempty"`
CreateResponse *bool `json:"create_response,omitempty"`
Eagerness *string `json:"eagerness,omitempty"`
SpeechDurationMs *int `json:"speech_duration_ms,omitempty"`
RemoveFillerWords *bool `json:"remove_filler_words,omitempty"`
InterruptResponse *bool `json:"interrupt_response,omitempty"`
Languages []string `json:"languages,omitempty"`
AutoTruncate *bool `json:"auto_truncate,omitempty"`
}

// VoiceTranscription enables user-speech transcription events on the input stream.
type VoiceTranscription struct {
Model string `json:"model,omitempty"`
Model string `json:"model,omitempty"`
Language *string `json:"language,omitempty"`
Prompt *string `json:"prompt,omitempty"`
}

// VoiceNoiseReduction configures input audio noise reduction.
type VoiceNoiseReduction struct {
Type string `json:"type"`
}

// VoiceInputConfig is the input (caller -> agent) audio configuration.
type VoiceInputConfig struct {
Format *VoiceAudioFormat `json:"format,omitempty"`
TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"`
Transcription *VoiceTranscription `json:"transcription,omitempty"`
Format *VoiceAudioFormat `json:"format,omitempty"`
NoiseReduction *VoiceNoiseReduction `json:"noise_reduction,omitempty"`
EchoCancellation map[string]any `json:"echo_cancellation,omitempty"`
TurnDetection *VoiceTurnDetection `json:"turn_detection,omitempty"`
Transcription *VoiceTranscription `json:"transcription,omitempty"`
}

// VoiceConfig selects the output voice. Type is "openai" for realtime voices
// (single lowercase word, e.g. "alloy") or "azure_standard" for Azure Neural
// voices (e.g. "en-US-Ava:DragonHDLatestNeural").
type VoiceConfig struct {
Type string `json:"type"`
Name string `json:"name"`
Type string `json:"type"`
Name string `json:"name"`
Style *string `json:"style,omitempty"`
Pitch *string `json:"pitch,omitempty"`
Rate *string `json:"rate,omitempty"`
Locale *string `json:"locale,omitempty"`
Volume *string `json:"volume,omitempty"`
}

// VoiceOutputConfig is the output (agent -> caller) audio configuration.
type VoiceOutputConfig struct {
Format *VoiceAudioFormat `json:"format,omitempty"`
Voice *VoiceConfig `json:"voice,omitempty"`
Speed *float64 `json:"speed,omitempty"`
}

// VoiceOutputConfigFlat is the newer Voice Live output shape used by the
// unified /agents voice API in TiP. Older regions still accept/return the
// object-shaped VoiceOutputConfig above.
type VoiceOutputConfigFlat struct {
Format *VoiceAudioFormat `json:"format,omitempty"`
Voice string `json:"voice,omitempty"`
VoiceType string `json:"voice_type,omitempty"`
VoiceLocale string `json:"voice_locale,omitempty"`
Style *string `json:"style,omitempty"`
Pitch *string `json:"pitch,omitempty"`
Rate *string `json:"rate,omitempty"`
Volume *string `json:"volume,omitempty"`
Speed *float64 `json:"speed,omitempty"`
}

// VoiceAudioConfig bundles the input and output audio configuration.
Expand All @@ -404,17 +441,54 @@ type VoiceAudioConfig struct {
Output *VoiceOutputConfig `json:"output,omitempty"`
}

// VoiceAudioConfigFlat bundles voice audio config with the flat output shape.
type VoiceAudioConfigFlat struct {
Input *VoiceInputConfig `json:"input,omitempty"`
Output *VoiceOutputConfigFlat `json:"output,omitempty"`
}

// VoiceAgentDefinition is the data-plane definition body POSTed to the
// /voice_agents collection for a declarative (managed) voice agent. Its Kind
// is always AgentKindVoice ("voice").
type VoiceAgentDefinition struct {
AgentDefinition
ModelType VoiceModelType `json:"model_type"`
Model string `json:"model"`
Instructions string `json:"instructions,omitempty"`
Audio *VoiceAudioConfig `json:"audio,omitempty"`
OutputModalities []string `json:"output_modalities,omitempty"`
Store *bool `json:"store,omitempty"`
ModelType VoiceModelType `json:"model_type"`
Model string `json:"model"`
Instructions string `json:"instructions,omitempty"`
StructuredInputs map[string]any `json:"structured_inputs,omitempty"`
Audio *VoiceAudioConfig `json:"audio,omitempty"`
OutputModalities []string `json:"output_modalities,omitempty"`
Store *bool `json:"store,omitempty"`
Tools []map[string]any `json:"tools,omitempty"`
Avatar map[string]any `json:"avatar,omitempty"`
Greeting map[string]any `json:"greeting,omitempty"`
Handoff map[string]any `json:"handoff,omitempty"`
ToolChoice any `json:"tool_choice,omitempty"`
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
MaxOutputTokens any `json:"max_output_tokens,omitempty"`
Include []string `json:"include,omitempty"`
}

// VoiceAgentDefinitionFlat is the voice definition shape aligned with the
// unified /agents TiP API, where audio.output.voice is a string and the voice
// provider details are sibling fields.
type VoiceAgentDefinitionFlat struct {
AgentDefinition
ModelType VoiceModelType `json:"model_type"`
Model string `json:"model"`
Instructions string `json:"instructions,omitempty"`
StructuredInputs map[string]any `json:"structured_inputs,omitempty"`
Audio *VoiceAudioConfigFlat `json:"audio,omitempty"`
OutputModalities []string `json:"output_modalities,omitempty"`
Store *bool `json:"store,omitempty"`
Tools []map[string]any `json:"tools,omitempty"`
Avatar map[string]any `json:"avatar,omitempty"`
Greeting map[string]any `json:"greeting,omitempty"`
Handoff map[string]any `json:"handoff,omitempty"`
ToolChoice any `json:"tool_choice,omitempty"`
ParallelToolCalls *bool `json:"parallel_tool_calls,omitempty"`
MaxOutputTokens any `json:"max_output_tokens,omitempty"`
Include []string `json:"include,omitempty"`
}

// CreateAgentVersionRequest represents a request to create an agent version
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,55 @@ func (c *AgentClient) CreateAgent(ctx context.Context, request *CreateAgentReque
// header while voice agents remain a preview capability.
const voiceAgentsPreviewFeature = "VoiceAgents=V1Preview"

func (c *AgentClient) doVoiceJSONAgentRequest(
ctx context.Context,
method string,
url string,
request any,
overriddenHost string,
) (*AgentObject, error) {
payload, err := json.Marshal(request)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}

req, err := runtime.NewRequest(ctx, method, url)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}

req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature)
if overriddenHost != "" {
req.Raw().Header.Set("x-ms-overridden-host", overriddenHost)
}

if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil {
return nil, fmt.Errorf("failed to set request body: %w", err)
}

resp, err := c.pipeline.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
}
defer resp.Body.Close()

if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated) {
return nil, runtime.NewResponseError(resp)
}

body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response body: %w", err)
}

var agent AgentObject
if err := json.Unmarshal(body, &agent); err != nil {
return nil, fmt.Errorf("failed to parse response: %w", err)
}

return &agent, nil
}

// CreateVoiceAgent creates a new declarative (managed) voice agent.
//
// Voice agents live in a separate data-plane collection (/voice_agents), distinct
Expand All @@ -193,36 +242,36 @@ func (c *AgentClient) CreateVoiceAgent(
overriddenHost string,
) (*AgentObject, error) {
url := fmt.Sprintf("%s/voice_agents?api-version=%s", c.endpoint, apiVersion)
return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost)
}

payload, err := json.Marshal(request)
if err != nil {
return nil, fmt.Errorf("failed to marshal request: %w", err)
}

req, err := runtime.NewRequest(ctx, http.MethodPost, url)
// GetVoiceAgentUnified retrieves a voice agent through the unified /agents
// endpoint with the voice preview opt-in header. Use this instead of GetAgent
// when deciding whether to create or update a prompt voice agent.
func (c *AgentClient) GetVoiceAgentUnified(
ctx context.Context,
agentName string,
apiVersion string,
overriddenHost string,
) (*AgentObject, error) {
url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion)
req, err := runtime.NewRequest(ctx, http.MethodGet, url)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}

// Voice agents are a preview feature; the service rejects the request with
// 403 preview_feature_required unless this opt-in header is present.
req.Raw().Header.Set("Foundry-Features", voiceAgentsPreviewFeature)

if overriddenHost != "" {
req.Raw().Header.Set("x-ms-overridden-host", overriddenHost)
}

if err := req.SetBody(streaming.NopCloser(bytes.NewReader(payload)), "application/json"); err != nil {
return nil, fmt.Errorf("failed to set request body: %w", err)
}

resp, err := c.pipeline.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP request failed: %w", err)
}
defer resp.Body.Close()

if !runtime.HasStatusCode(resp, http.StatusOK, http.StatusCreated) {
if !runtime.HasStatusCode(resp, http.StatusOK) {
return nil, runtime.NewResponseError(resp)
}

Expand All @@ -239,6 +288,31 @@ func (c *AgentClient) CreateVoiceAgent(
return &agent, nil
}

// CreateVoiceAgentUnified creates a voice agent through the unified /agents
// collection. This path is opt-in while regional rollout is still in progress.
func (c *AgentClient) CreateVoiceAgentUnified(
ctx context.Context,
request *CreateAgentRequest,
apiVersion string,
overriddenHost string,
) (*AgentObject, error) {
url := fmt.Sprintf("%s/agents?api-version=%s", c.endpoint, apiVersion)
return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost)
}

// UpdateVoiceAgentUnified creates a new version for an existing voice agent
// through the unified /agents/{name} endpoint.
func (c *AgentClient) UpdateVoiceAgentUnified(
ctx context.Context,
agentName string,
request *UpdateAgentRequest,
apiVersion string,
overriddenHost string,
) (*AgentObject, error) {
url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion)
return c.doVoiceJSONAgentRequest(ctx, http.MethodPost, url, request, overriddenHost)
}

// UpdateAgent updates an existing agent
func (c *AgentClient) UpdateAgent(ctx context.Context, agentName string, request *UpdateAgentRequest, apiVersion string) (*AgentObject, error) {
url := fmt.Sprintf("%s/agents/%s?api-version=%s", c.endpoint, agentName, apiVersion)
Expand Down
Loading
Loading