Rhidp 14314/attachment type model validation

workspaces/intelligent-assistant/.changeset/clean-jokes-wonder.md

@@ -0,0 +1,5 @@
+---
+'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': minor
+---
+
+add model-validation route to verify whether a model is multimodal. Can now attach images to models that support it.

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/attachment-validation.test.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ModelCapabilitiesCache } from './attachment-validation';
+
+describe('ModelCapabilitiesCache', () => {
+  beforeEach(() => {
+    ModelCapabilitiesCache.clear();
+  });
+
+  it('should store and retrieve model capabilities', () => {
+    ModelCapabilitiesCache.set('gpt-4-vision', true);
+    expect(ModelCapabilitiesCache.get('gpt-4-vision')).toBe(true);
+    expect(ModelCapabilitiesCache.has('gpt-4-vision')).toBe(true);
+  });
+
+  it('should return undefined for unknown models', () => {
+    expect(ModelCapabilitiesCache.get('unknown-model')).toBeUndefined();
+    expect(ModelCapabilitiesCache.has('unknown-model')).toBe(false);
+  });
+
+  it('should update existing entries', () => {
+    ModelCapabilitiesCache.set('gpt-4', false);
+    expect(ModelCapabilitiesCache.get('gpt-4')).toBe(false);
+
+    ModelCapabilitiesCache.set('gpt-4', true);
+    expect(ModelCapabilitiesCache.get('gpt-4')).toBe(true);
+  });
+
+  it('should handle multiple models independently', () => {
+    ModelCapabilitiesCache.set('model-a', true);
+    ModelCapabilitiesCache.set('model-b', false);
+    ModelCapabilitiesCache.set('model-c', true);
+
+    expect(ModelCapabilitiesCache.get('model-a')).toBe(true);
+    expect(ModelCapabilitiesCache.get('model-b')).toBe(false);
+    expect(ModelCapabilitiesCache.get('model-c')).toBe(true);
+    expect(ModelCapabilitiesCache.has('model-a')).toBe(true);
+    expect(ModelCapabilitiesCache.has('model-b')).toBe(true);
+    expect(ModelCapabilitiesCache.has('model-c')).toBe(true);
+    expect(ModelCapabilitiesCache.has('model-d')).toBe(false);
+  });
+
+  it('should expire entries after TTL', () => {
+    ModelCapabilitiesCache.set('model-ttl', true);
+    expect(ModelCapabilitiesCache.get('model-ttl')).toBe(true);
+
+    const entry = ModelCapabilitiesCache.cache['model-ttl'];
+    entry.expiry = Date.now() - 1;
+
+    expect(ModelCapabilitiesCache.get('model-ttl')).toBeUndefined();
+    expect(ModelCapabilitiesCache.has('model-ttl')).toBe(false);
+  });
+});

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/attachment-validation.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
+
+export const ModelCapabilitiesCache = {
+  cache: {} as Record<string, { value: boolean; expiry: number }>,
+
+  get(model: string): boolean | undefined {
+    const entry = this.cache[model];
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiry) {
+      delete this.cache[model];
+      return undefined;
+    }
+    return entry.value;
+  },
+
+  set(model: string, supportsVision: boolean): void {
+    this.cache[model] = {
+      value: supportsVision,
+      expiry: Date.now() + CACHE_TTL_MS,
+    };
+  },
+
+  has(model: string): boolean {
+    return this.get(model) !== undefined;
+  },
+
+  clear(): void {
+    this.cache = {};
+  },
+};

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/constant.ts

@@ -214,3 +214,14 @@ export const SKIP_USER_ID_ENDPOINTS = new Set(['/v1/models', '/v1/shields']);
 
 // default number of message history being loaded
 export const DEFAULT_HISTORY_LENGTH = 10;
+
+/**
+ * Minimal 1x1 pixel JPEG image for testing model vision capabilities.
+ * Base64-encoded JPEG with minimal headers.
+ */
+export const TEST_VISION_JPEG =
+  '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0a' +
+  'HBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIy' +
+  'MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAABAAEDASIA' +
+  'AhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEB' +
+  'AQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCwAA//2Q==';

workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts

@@ -29,6 +29,7 @@ import request from 'supertest';
 import { handlers, LOCAL_AI_ADDR } from '../../__fixtures__/handlers';
 import { lcsHandlers, LOCAL_LCS_ADDR } from '../../__fixtures__/lcsHandlers';
 import { intelligentAssistantPlugin } from '../plugin';
+import { ModelCapabilitiesCache } from './attachment-validation';
 import { VectorStoresOperator } from './notebooks/VectorStoresOperator';
 
 const mockUserId = `user: default/user1`;
@@ -1356,4 +1357,192 @@ describe('intelligent-assistant router tests', () => {
       expect(modelsResponse.statusCode).toBe(200);
     });
   });
+
+  describe('POST /v1/validate-model-vision', () => {
+    beforeEach(() => {
+      ModelCapabilitiesCache.clear();
+    });
+
+    it('returns true when model supports vision', async () => {
+      server.use(
+        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => {
+          return HttpResponse.json({ id: 'resp-1', output: [] });
+        }),
+      );
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/validate-model-vision')
+        .send({ model: 'gpt-4o', provider: 'test-server' });
+
+      expect(response.statusCode).toEqual(200);
+      expect(response.body).toEqual({
+        model: 'gpt-4o',
+        provider: 'test-server',
+        supportsVision: true,
+      });
+    });
+
+    it('returns false when model lacks vision', async () => {
+      server.use(
+        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => {
+          return new HttpResponse(
+            JSON.stringify({ error: 'Model does not support vision' }),
+            { status: 400 },
+          );
+        }),
+      );
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/validate-model-vision')
+        .send({ model: 'gpt-3.5-turbo', provider: 'test-server' });
+
+      expect(response.statusCode).toEqual(200);
+      expect(response.body).toEqual({
+        model: 'gpt-3.5-turbo',
+        provider: 'test-server',
+        supportsVision: false,
+      });
+    });
+
+    it('returns false when model is not found', async () => {
+      server.use(
+        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => {
+          return new HttpResponse(
+            JSON.stringify({ error: 'Model not found' }),
+            { status: 404 },
+          );
+        }),
+      );
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/validate-model-vision')
+        .send({ model: 'gpt-4o', provider: 'test-server' });
+
+      expect(response.statusCode).toEqual(200);
+      expect(response.body).toEqual({
+        model: 'gpt-4o',
+        provider: 'test-server',
+        supportsVision: false,
+      });
+    });
+
+    it('returns 502 without caching when upstream returns 5xx', async () => {
+      server.use(
+        http.post(`${LOCAL_LCS_ADDR}/v1/responses`, () => {
+          return new HttpResponse(
+            JSON.stringify({ error: 'Internal server error' }),
+            { status: 500 },
+          );
+        }),
+      );
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/validate-model-vision')
+        .send({ model: 'gpt-4o', provider: 'test-server' });
+
+      expect(response.statusCode).toEqual(502);
+      expect(response.body.error).toContain('Unable to verify vision support');
+      expect(ModelCapabilitiesCache.has('test-server/gpt-4o')).toBe(false);
+    });
+
+    it('returns 400 when model or provider is missing', async () => {
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/validate-model-vision')
+        .send({ model: 'gpt-4o' });
+
+      expect(response.statusCode).toEqual(400);
+      expect(response.body).toEqual({
+        error: 'model and provider are required',
+      });
+    });
+  });
+
+  describe('POST /v1/query attachment validation', () => {
+    const VALID_JPEG_B64 = Buffer.from([
+      0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10,
+    ]).toString('base64');
+
+    beforeEach(() => {
+      ModelCapabilitiesCache.clear();
+    });
+
+    it('rejects attachments when model lacks vision', async () => {
+      ModelCapabilitiesCache.set('test-server/gpt-3.5-turbo', false);
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/query')
+        .send({
+          model: 'gpt-3.5-turbo',
+          provider: 'test-server',
+          query: 'What is this?',
+          attachments: [
+            {
+              attachment_type: 'image',
+              content_type: 'image/jpeg',
+              content: VALID_JPEG_B64,
+            },
+          ],
+        });
+
+      expect(response.statusCode).toEqual(400);
+      expect(response.body.error).toContain(
+        'This model does not support JPEG images',
+      );
+    });
+
+    it('accepts attachments when model supports vision', async () => {
+      ModelCapabilitiesCache.set('test-server/gpt-4o', true);
+
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/query')
+        .send({
+          model: 'gpt-4o',
+          provider: 'test-server',
+          query: 'What is this?',
+          attachments: [
+            {
+              attachment_type: 'image',
+              content_type: 'image/jpeg',
+              content: VALID_JPEG_B64,
+            },
+          ],
+        });
+
+      expect(response.statusCode).toEqual(200);
+    });
+
+    it('accepts empty attachments regardless of vision support', async () => {
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/query')
+        .send({
+          model: 'gpt-3.5-turbo',
+          provider: 'test-server',
+          query: 'Hello',
+          attachments: [],
+        });
+
+      expect(response.statusCode).toEqual(200);
+    });
+
+    it('accepts no attachments field regardless of vision support', async () => {
+      const backendServer = await startBackendServer();
+      const response = await request(backendServer)
+        .post('/api/intelligent-assistant/v1/query')
+        .send({
+          model: 'gpt-3.5-turbo',
+          provider: 'test-server',
+          query: 'Hello',
+        });
+
+      expect(response.statusCode).toEqual(200);
+    });
+  });
 });