Skip to content

Fix multi-endpoint Vision model menu and test connection #986

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paullizer merged 4 commits into from
Jul 2, 2026
Merged

Fix multi-endpoint Vision model menu and test connection #986

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
043fad2
update-vision-model-section - Fixed multi-modal vision analysis menu.
chadpalmer Jun 30, 2026
9f0c6d3
Merge branch 'Development' into update-vision-model-section
chadpalmer Jul 1, 2026
80f6975
update-vision-model-section - Fixed multi-modal vision test button.
chadpalmer Jul 1, 2026
6577e6d
update-vision-model-section - Added documentation.
chadpalmer Jul 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion application/single_app/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@
EXECUTOR_TYPE = 'thread'
EXECUTOR_MAX_WORKERS = 30
SESSION_TYPE = 'filesystem'
VERSION = "0.250.007"
VERSION = "0.250.008"

SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')
SESSION_COOKIE_HTTPONLY = os.getenv('SESSION_COOKIE_HTTPONLY', 'true').lower() != 'false'
Expand Down
152 changes: 67 additions & 85 deletions application/single_app/route_backend_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
from functions_settings import *
from functions_web_search_test import run_web_search_connection_test
from functions_url_access_policy_test import run_url_access_policy_test
from functions_model_endpoint_runtime import (

Check warning on line 9 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
build_model_endpoint_sync_chat_client,

Check warning on line 10 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
resolve_model_endpoint_from_context,

Check warning on line 11 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
)
from functions_activity_logging import (
log_admin_feedback_email_submission,
log_general_admin_action,
Expand Down Expand Up @@ -85,6 +89,8 @@
'web_search_agent.other_settings.azure_ai_foundry.client_secret',
)
elif test_type == 'multimodal_vision':
if isinstance(payload.get('multi_endpoint'), dict):
return payload
if payload.get('enable_apim'):
_resolve_test_payload_secret(payload, ('apim', 'subscription_key'), settings, 'azure_apim_gpt_subscription_key')
else:
Expand Down Expand Up @@ -508,9 +514,6 @@

elif test_type == 'key_vault':
return _test_key_vault_connection(data)

elif test_type == 'multimodal_vision':
return _test_multimodal_vision_connection(data)

else:
return jsonify({'error': f'Unknown test_type: {test_type}'}), 400
Expand Down Expand Up @@ -983,86 +986,7 @@
"""Test multi-modal vision analysis with a sample image."""
enable_apim = payload.get('enable_apim', False)
vision_model = payload.get('vision_model')

if not vision_model:
return jsonify({'error': 'No vision model specified'}), 400

# Create a simple test image (1x1 red pixel PNG)
test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="

try:
if enable_apim:
apim_data = payload.get('apim', {})
endpoint = apim_data.get('endpoint')
api_version = apim_data.get('api_version')
subscription_key = apim_data.get('subscription_key')

gpt_client = AzureOpenAI(
api_version=api_version,
azure_endpoint=endpoint,
api_key=subscription_key
)
else:
direct_data = payload.get('direct', {})
endpoint = direct_data.get('endpoint')
api_version = direct_data.get('api_version')
auth_type = direct_data.get('auth_type', 'key')

if auth_type == 'managed_identity':
token_provider = get_bearer_token_provider(
DefaultAzureCredential(),
cognitive_services_scope
)
gpt_client = AzureOpenAI(
api_version=api_version,
azure_endpoint=endpoint,
azure_ad_token_provider=token_provider
)
else:
api_key = direct_data.get('key')
gpt_client = AzureOpenAI(
api_version=api_version,
azure_endpoint=endpoint,
api_key=api_key
)

# Test vision analysis with simple prompt
response = gpt_client.chat.completions.create(
model=vision_model,
messages=[
{
"role": "user",
"content": [
{
"type": "text",
"text": "What color is this image? Just say the color."
},
{
"type": "image_url",
"image_url": {
"url": f"data:image/png;base64,{test_image_base64}"
}
}
]
}
],
max_tokens=50
)

result = response.choices[0].message.content

return jsonify({
'message': 'Multi-modal vision connection successful',
'details': f'Model responded: {result}'
}), 200

except Exception as e:
return jsonify({'error': f'Vision test failed: {str(e)}'}), 500

def _test_multimodal_vision_connection(payload):
"""Test multi-modal vision analysis with a sample image."""
enable_apim = payload.get('enable_apim', False)
vision_model = payload.get('vision_model')
vision_model_name = payload.get('model_name') or vision_model

if not vision_model:
return jsonify({'error': 'No vision model specified'}), 400
Expand All @@ -1071,7 +995,57 @@
test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="

try:
if enable_apim:
multi_endpoint_selection = payload.get('multi_endpoint') if isinstance(payload.get('multi_endpoint'), dict) else None
if multi_endpoint_selection:
settings = get_settings()

Check warning on line 1000 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.
model_context = {
'endpoint_id': str(multi_endpoint_selection.get('endpoint_id') or '').strip(),
'model_id': str(multi_endpoint_selection.get('model_id') or '').strip(),
'provider': str(multi_endpoint_selection.get('provider') or '').strip(),
'model_deployment': str(
multi_endpoint_selection.get('deployment_name')
or vision_model
or ''
).strip(),
}
resolved_endpoint = resolve_model_endpoint_from_context(settings, model_context)

Check warning on line 1011 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
if not resolved_endpoint:
return jsonify({'error': 'Selected vision model endpoint could not be resolved from saved settings'}), 400

resolved_models = resolved_endpoint.get('models', []) or []
matched_model = next(
(
model for model in resolved_models
if str(model.get('id') or '').strip() == model_context['model_id']
),
None,
)
if not matched_model:
matched_model = next(
(
model for model in resolved_models
if str(model.get('deploymentName') or model.get('deployment') or '').strip() == model_context['model_deployment']
),
None,
)
if not matched_model:
return jsonify({'error': 'Selected vision model could not be resolved from saved settings'}), 400

vision_model = str(
matched_model.get('deploymentName')
or matched_model.get('deployment')
or model_context['model_deployment']
).strip()
vision_model_name = str(matched_model.get('modelName') or vision_model).strip()
connection = resolved_endpoint.get('connection', {}) or {}
gpt_client, _ = build_model_endpoint_sync_chat_client(

Check warning on line 1041 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains AI, plugin, agent, or workspace boundary marker. Recommendation%3A Check whether prompts, chat history, uploaded documents, embeddings, citations, settings, or identity can cross a new boundary.
resolved_endpoint.get('auth', {}) or {},
resolved_endpoint.get('provider') or model_context.get('provider') or 'aoai',
connection.get('endpoint'),
connection.get('openai_api_version') or connection.get('api_version'),

Check warning on line 1045 in application/single_app/route_backend_settings.py

View workflow job for this annotation

GitHub Actions / malicious-pr-security-review 

Important - Changed line contains secret or sensitive data source marker. Recommendation%3A Pair this source with any nearby network, logging, serialization, or process execution sink before approving.
deployment_name=vision_model,
)
elif enable_apim:
apim_data = payload.get('apim', {})
endpoint = apim_data.get('endpoint')
api_version = apim_data.get('api_version')
Expand Down Expand Up @@ -1109,6 +1083,7 @@
# Determine which token parameter to use based on model type
# o-series and gpt-5 models require max_completion_tokens instead of max_tokens
vision_model_lower = vision_model.lower()
vision_model_name_lower = vision_model_name.lower()
api_params = {
"model": vision_model,
"messages": [
Expand All @@ -1131,7 +1106,14 @@
}

# Use max_completion_tokens for o-series and gpt-5 models, max_tokens for others
if ('o1' in vision_model_lower or 'o3' in vision_model_lower or 'gpt-5' in vision_model_lower):
if (
'o1' in vision_model_lower or
'o3' in vision_model_lower or
'gpt-5' in vision_model_lower or
'o1' in vision_model_name_lower or
'o3' in vision_model_name_lower or
'gpt-5' in vision_model_name_lower
):
api_params["max_completion_tokens"] = 50
else:
api_params["max_tokens"] = 50
Expand Down
107 changes: 69 additions & 38 deletions application/single_app/static/js/admin/admin_settings.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6072,13 +6072,24 @@ function setupTestButtons() {
}

const enableApim = document.getElementById('enable_gpt_apim').checked;
const selectedVisionOption = getSelectedVisionModelOption();

const payload = {
test_type: 'multimodal_vision',
enable_apim: enableApim,
vision_model: visionModel
};

if (!enableApim && selectedVisionOption?.dataset?.endpointId && selectedVisionOption?.dataset?.modelId) {
payload.multi_endpoint = {
endpoint_id: selectedVisionOption.dataset.endpointId,
model_id: selectedVisionOption.dataset.modelId,
provider: selectedVisionOption.dataset.provider || '',
model_name: selectedVisionOption.dataset.modelName || '',
deployment_name: visionModel
};
}

if (enableApim) {
payload.apim = {
endpoint: document.getElementById('azure_apim_gpt_endpoint').value,
Expand Down Expand Up @@ -6951,56 +6962,76 @@ const visionToggle = document.getElementById('enable_multimodal_vision');
const visionModelDiv = document.getElementById('multimodal_vision_model_settings');
const visionSelect = document.getElementById('multimodal_vision_model');

function isVisionCapableModelName(modelName) {
const modelNameLower = (modelName || '').toLowerCase();
return (
modelNameLower.includes('vision') ||
modelNameLower.includes('gpt-4o') ||
modelNameLower.includes('gpt-4.1') ||
modelNameLower.includes('gpt-4.5') ||
modelNameLower.includes('gpt-5') ||
/^o\d+/.test(modelNameLower) ||
modelNameLower.includes('o1-') ||
modelNameLower.includes('o3-')
);
}

function getSelectedVisionModelOption() {
if (!visionSelect) {
return null;
}

return visionSelect.options[visionSelect.selectedIndex] || null;
}

function populateVisionModels() {
if (!visionSelect) return;

// remember previously chosen value
const prev = visionSelect.getAttribute('data-prev') || '';

// clear out old options (except the placeholder)
const prev = visionSelect.getAttribute('data-prev') || '';
visionSelect.innerHTML = '<option value="">Select a vision-capable model...</option>';

if (document.getElementById('enable_gpt_apim').checked) {
// use comma-separated APIM deployments
const text = document.getElementById('azure_apim_gpt_deployment').value || '';
// Prefer multi-endpoint model list when available
const multiEnabled = window.enableMultiModelEndpoints === true;
const endpoints = Array.isArray(window.modelEndpoints) ? window.modelEndpoints : [];

if (multiEnabled && endpoints.length > 0) {
endpoints
.filter(ep => ep && ep.enabled)
.forEach(ep => {
(ep.models || [])
.filter(m => m && m.enabled && isVisionCapableModelName(m.modelName || m.displayName))
.forEach(m => {
const value = m.deploymentName;
const label = `${m.displayName || m.deploymentName} (${m.modelName})`;
const opt = new Option(label, value);
opt.dataset.endpointId = ep.id || '';
opt.dataset.modelId = m.id || '';
opt.dataset.provider = ep.provider || '';
opt.dataset.modelName = m.modelName || '';
visionSelect.add(opt);
});
});
} else if (document.getElementById('enable_gpt_apim') && document.getElementById('enable_gpt_apim').checked) {
// Legacy APIM-based GPT configuration
const text = (document.getElementById('azure_apim_gpt_deployment')?.value || '');
text.split(',')
.map(s => s.trim())
.filter(s => s)
.forEach(d => {
const opt = new Option(d, d);
visionSelect.add(opt);
});
.map(s => s.trim())
.filter(s => s && isVisionCapableModelName(s))
.forEach(d => {
const opt = new Option(d, d);
visionSelect.add(opt);
});
} else {
// use direct GPT selected deployments - filter for vision-capable models
(window.gptSelected || []).forEach(m => {
// Only include models with vision capabilities
// Vision-enabled models per Azure OpenAI docs:
// - o-series reasoning models (o1, o3, etc.)
// - GPT-5 series
// - GPT-4.1 series
// - GPT-4.5
// - GPT-4o series (gpt-4o, gpt-4o-mini)
// - GPT-4 vision models (gpt-4-vision, gpt-4-turbo-vision)
const modelNameLower = (m.modelName || '').toLowerCase();
const isVisionCapable =
modelNameLower.includes('vision') ||
modelNameLower.includes('gpt-4o') ||
modelNameLower.includes('gpt-4.1') ||
modelNameLower.includes('gpt-4.5') ||
modelNameLower.includes('gpt-5') ||
modelNameLower.match(/^o\d+/) ||
modelNameLower.includes('o1-') ||
modelNameLower.includes('o3-');

if (isVisionCapable) {
// Legacy single-endpoint GPT configuration using window.gptSelected
(window.gptSelected || [])
.filter(m => isVisionCapableModelName(m.modelName))
.forEach(m => {
const label = `${m.deploymentName} (${m.modelName})`;
const opt = new Option(label, m.deploymentName);
visionSelect.add(opt);
}
});
});
}

// restore previous
if (prev) {
visionSelect.value = prev;
}
Expand Down
Loading
Loading