diff --git a/.changeset/openai-compatible-provider.md b/.changeset/openai-compatible-provider.md new file mode 100644 index 000000000..3a63453cd --- /dev/null +++ b/.changeset/openai-compatible-provider.md @@ -0,0 +1,7 @@ +--- +'@roomote/web': minor +'@roomote/types': minor +'@roomote/worker': minor +--- + +Add an OpenAI-compatible inference provider option for any OpenAI API endpoint. diff --git a/apps/docs/docs.json b/apps/docs/docs.json index 7f5e57d33..330a436b5 100644 --- a/apps/docs/docs.json +++ b/apps/docs/docs.json @@ -45,6 +45,7 @@ "root": "models", "expanded": true, "pages": [ + "providers/inference/openai-compatible", "providers/inference/litellm", "providers/inference/ollama", "providers/inference/vllm" diff --git a/apps/docs/environment-variables.mdx b/apps/docs/environment-variables.mdx index 01f5c2b66..610f11eda 100644 --- a/apps/docs/environment-variables.mdx +++ b/apps/docs/environment-variables.mdx @@ -192,6 +192,8 @@ as per-task auth tokens or workspace paths. | `LITELLM_BASE_URL` | LiteLLM | LiteLLM endpoint URL, usually including its `/v1` path. | | `LITELLM_API_KEY` | LiteLLM | LiteLLM gateway API key. Required when configuring LiteLLM. | | `OLLAMA_BASE_URL` | Ollama | Ollama endpoint URL. Use the service root, for example `http://ollama:11434`. | +| `OPENAI_COMPATIBLE_BASE_URL` | OpenAI-compatible | Generic OpenAI API endpoint URL, usually including its `/v1` path. | +| `OPENAI_COMPATIBLE_API_KEY` | Optional | Bearer API key for an OpenAI-compatible endpoint that requires authentication. | | `VLLM_BASE_URL` | vLLM | vLLM OpenAI-compatible endpoint URL, usually including its `/v1` path. | | `VLLM_API_KEY` | Optional | Bearer API key for a vLLM endpoint that requires authentication. | diff --git a/apps/docs/models.mdx b/apps/docs/models.mdx index 0e09323a7..ac46568c5 100644 --- a/apps/docs/models.mdx +++ b/apps/docs/models.mdx @@ -40,14 +40,16 @@ a provider never re-adds models you removed. ### Self-hosted and gateway providers -LiteLLM, Ollama, and vLLM let you bring an endpoint that you operate or host on -your own infrastructure. Their models are discovered from the configured -endpoint instead of Roomote's recommended catalog. Connect the provider in -**Settings > Models**, then select from the discovered models and choose the -defaults and role mappings that fit your deployment. +OpenAI-compatible endpoints let you bring any server that speaks the OpenAI +`/v1` API — including LiteLLM, Ollama, vLLM, and custom proxies. Models are +discovered from the configured endpoint instead of Roomote's recommended +catalog. Connect the provider in **Settings > Models**, then select from the +discovered models and choose the defaults and role mappings that fit your +deployment. | Provider | Best for | Setup | | --- | --- | --- | +| [OpenAI-compatible](/providers/inference/openai-compatible) | Any OpenAI API endpoint (LiteLLM, vLLM, custom proxies, local servers) | Endpoint URL and optional API key | | [LiteLLM](/providers/inference/litellm) | A single gateway that routes to one or more model vendors | Endpoint URL and required API key | | [Ollama](/providers/inference/ollama) | Local or private model serving without provider API keys | Endpoint URL | | [vLLM](/providers/inference/vllm) | Serving OpenAI-compatible models on your own GPU infrastructure | Endpoint URL and optional API key | diff --git a/apps/docs/providers/inference/openai-compatible.mdx b/apps/docs/providers/inference/openai-compatible.mdx new file mode 100644 index 000000000..7510fbd4f --- /dev/null +++ b/apps/docs/providers/inference/openai-compatible.mdx @@ -0,0 +1,67 @@ +--- +title: OpenAI-compatible +icon: plug +description: Connect any OpenAI API endpoint as a Roomote inference provider. +--- + +OpenAI-compatible is the generic option for any server that implements the +OpenAI `/v1` API. Use it for LiteLLM proxies, vLLM, local OpenAI servers, or +other OpenAI-shaped gateways when you do not need a provider-specific setup page. + +## Configure an OpenAI-compatible endpoint + +In **Settings > Models**, add **OpenAI-compatible** and provide the endpoint URL. +An API key is optional and only needed when your endpoint requires bearer auth. +You can instead manage both values as deployment environment variables: + +```sh +OPENAI_COMPATIBLE_BASE_URL=https://proxy.example.com/v1 +OPENAI_COMPATIBLE_API_KEY=... +``` + +The endpoint must be reachable from the Roomote deployment. For a Compose +deployment, use a service DNS name such as `http://litellm:4000/v1` when the +proxy runs on the same private network. Do not use `localhost` unless the +endpoint runs in the same network namespace as the Roomote service that proxies +inference requests. + +After saving the provider, Roomote discovers models from `/v1/models`. Enable +the models you want and select a default coding model and any specialized roles. +Model IDs use the `openai-compatible/` form, where `` is the +name returned by your endpoint. + +## When to use this versus LiteLLM, Ollama, or vLLM + +Choose **OpenAI-compatible** when you want one clear setup path for any OpenAI +API endpoint. Choose the named LiteLLM, Ollama, or vLLM providers when you +prefer those labels or their dedicated env var names. Runtime discovery and +chat completion behavior are the same OpenAI `/v1` protocol for all of them. + +## Secure the endpoint + +Keep the endpoint private to Roomote whenever possible. Put it on an internal +network or behind a private ingress, require an API key when the server supports +it, and use TLS when traffic crosses an untrusted network. Store +`OPENAI_COMPATIBLE_API_KEY` in your deployment secret manager or as an encrypted +Roomote deployment variable, not in an environment's task variables or +repository files. + +Roomote proxies model traffic through its inference gateway, so task sandboxes +do not need direct network access to the endpoint or the API key. + +## Verify setup + +1. save the endpoint URL and optional API key +2. confirm models appear in **Settings > Models** +3. enable one model and make it the default coding model +4. start a small Roomote task and confirm it completes through the endpoint +5. check the upstream server logs for the request + +## Common issues + +- **No models appear.** Confirm the endpoint URL includes the OpenAI `/v1` API + path and that the API key can list models when auth is required. +- **Tasks cannot reach the endpoint.** Check DNS, container networking, firewall + rules, and whether the endpoint is reachable from the Roomote deployment. +- **Tool calling fails.** Roomote needs models that support tool calling. Confirm + the upstream model and gateway configuration expose tools over chat completions. diff --git a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx index e688038b4..54fd9c86c 100644 --- a/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx +++ b/apps/web/src/app/(onboarding)/setup/StepInferenceProvider.tsx @@ -209,13 +209,19 @@ export function StepInferenceProvider({ : apiKey.trim() || undefined; if (isEndpointProvider) { - const provider = selectedProvider as 'ollama' | 'vllm' | 'litellm'; + const provider = selectedProvider as + | 'openai-compatible' + | 'ollama' + | 'vllm' + | 'litellm'; + const endpointApiKeyEnvVarName = additionalEnvFields.find( + (field) => field.secret, + )?.envVarName; const connection = { baseUrl: submittedCredential, - apiKey: - additionalEnvValues[ - `${selectedProvider.toUpperCase()}_API_KEY` - ]?.trim() || undefined, + apiKey: endpointApiKeyEnvVarName + ? additionalEnvValues[endpointApiKeyEnvVarName]?.trim() || undefined + : undefined, }; const discovery = await discoverProviderModels.mutateAsync({ provider, diff --git a/apps/web/src/trpc/commands/task-models/index.test.ts b/apps/web/src/trpc/commands/task-models/index.test.ts index 3976803c8..82234382f 100644 --- a/apps/web/src/trpc/commands/task-models/index.test.ts +++ b/apps/web/src/trpc/commands/task-models/index.test.ts @@ -1259,6 +1259,7 @@ describe('task model provider commands', () => { 'AWS_REGION', 'LITELLM_BASE_URL', 'OLLAMA_BASE_URL', + 'OPENAI_COMPATIBLE_BASE_URL', 'VLLM_BASE_URL', ]); mockGetPersistedEnvironmentVariableValues.mockResolvedValue({ @@ -1269,6 +1270,7 @@ describe('task model provider commands', () => { expect(mockGetPersistedEnvironmentVariableValues).toHaveBeenCalledWith([ 'AWS_REGION', + 'OPENAI_COMPATIBLE_BASE_URL', 'LITELLM_BASE_URL', 'OLLAMA_BASE_URL', 'VLLM_BASE_URL', diff --git a/apps/web/src/trpc/commands/task-models/local-provider-discovery.test.ts b/apps/web/src/trpc/commands/task-models/local-provider-discovery.test.ts index ed6d7ebd5..090b5d0a3 100644 --- a/apps/web/src/trpc/commands/task-models/local-provider-discovery.test.ts +++ b/apps/web/src/trpc/commands/task-models/local-provider-discovery.test.ts @@ -19,6 +19,7 @@ describe('LOCAL_TASK_MODEL_PROVIDER_IDS', () => { expect([...LOCAL_TASK_MODEL_PROVIDER_IDS].sort()).toEqual([ 'litellm', 'ollama', + 'openai-compatible', 'vllm', ]); }); diff --git a/apps/web/src/trpc/commands/task-models/local-provider-discovery.ts b/apps/web/src/trpc/commands/task-models/local-provider-discovery.ts index a4ff2825a..814a469ea 100644 --- a/apps/web/src/trpc/commands/task-models/local-provider-discovery.ts +++ b/apps/web/src/trpc/commands/task-models/local-provider-discovery.ts @@ -27,10 +27,13 @@ const LOCAL_ENDPOINT_PROVIDER_CATALOG = SETUP_MODEL_PROVIDER_CATALOG.filter( ); export const LOCAL_TASK_MODEL_PROVIDER_IDS = - LOCAL_ENDPOINT_PROVIDER_CATALOG.map((provider) => provider.id); + LOCAL_ENDPOINT_PROVIDER_CATALOG.map((provider) => provider.id) as [ + (typeof LOCAL_ENDPOINT_PROVIDER_CATALOG)[number]['id'], + ...(typeof LOCAL_ENDPOINT_PROVIDER_CATALOG)[number]['id'][], + ]; export type LocalTaskModelProviderId = - (typeof LOCAL_ENDPOINT_PROVIDER_CATALOG)[number]['id']; + (typeof LOCAL_TASK_MODEL_PROVIDER_IDS)[number]; export type LocalProviderConnectionInput = { baseUrl?: string; diff --git a/apps/web/src/trpc/routers/_app.ts b/apps/web/src/trpc/routers/_app.ts index f3021691e..a2e6f006d 100644 --- a/apps/web/src/trpc/routers/_app.ts +++ b/apps/web/src/trpc/routers/_app.ts @@ -307,6 +307,7 @@ import { suggestTaskModelsCommand, updateTaskModelSettingsCommand, } from '../commands/task-models'; +import { LOCAL_TASK_MODEL_PROVIDER_IDS } from '../commands/task-models/local-provider-discovery'; import { disconnectChatGptSubscriptionCommand, getChatGptSubscriptionStatusCommand, @@ -1708,7 +1709,7 @@ export const appRouter = createRouter({ discoverProviderModels: protectedProcedure .input( z.object({ - provider: z.enum(['ollama', 'vllm', 'litellm']), + provider: z.enum(LOCAL_TASK_MODEL_PROVIDER_IDS), baseUrl: z.string().trim().optional(), apiKey: z.string().trim().optional(), }), @@ -1720,7 +1721,7 @@ export const appRouter = createRouter({ qualifyProviderModel: protectedProcedure .input( z.object({ - provider: z.enum(['ollama', 'vllm', 'litellm']), + provider: z.enum(LOCAL_TASK_MODEL_PROVIDER_IDS), modelId: z.string().trim().min(1), baseUrl: z.string().trim().optional(), apiKey: z.string().trim().optional(), diff --git a/apps/worker/src/run-task/agent-home.test.ts b/apps/worker/src/run-task/agent-home.test.ts index 8a2801671..2d83cbf86 100644 --- a/apps/worker/src/run-task/agent-home.test.ts +++ b/apps/worker/src/run-task/agent-home.test.ts @@ -319,10 +319,13 @@ describe('generateOpenCodeConfig provider support', () => { R_MODEL: 'ollama/qwen3-coder', R_SMALL_MODEL: 'vllm/meta-llama/Llama-3.3-70B-Instruct', R_VISION_MODEL: 'litellm/gpt-4.1-mini', + R_CODE_REVIEW_MODEL: 'openai-compatible/gpt-4o', VLLM_BASE_URL: 'https://vllm.example.com/v1', VLLM_API_KEY: 'vllm-key', LITELLM_BASE_URL: 'https://litellm.example.com/v1', LITELLM_API_KEY: 'litellm-key', + OPENAI_COMPATIBLE_BASE_URL: 'https://proxy.example.com/v1', + OPENAI_COMPATIBLE_API_KEY: 'compat-key', }, }); const config = JSON.parse(result.configContent) as { @@ -347,6 +350,58 @@ describe('generateOpenCodeConfig provider support', () => { apiKey: '{env:LITELLM_API_KEY}', }, }); + expect(config.provider['openai-compatible']).toMatchObject({ + npm: '@ai-sdk/openai-compatible', + options: { + baseURL: 'https://proxy.example.com/v1', + apiKey: '{env:OPENAI_COMPATIBLE_API_KEY}', + }, + models: { 'gpt-4o': { name: 'gpt-4o' } }, + }); + }); + + it('does not fall back openai-compatible to OPENAI_* credentials', () => { + const result = generateOpenCodeConfig({ + homeDir: createHomeDir(), + runtimeEnv: { + R_MODEL: 'openai-compatible/gpt-4o', + OPENAI_COMPATIBLE_BASE_URL: 'https://proxy.example.com/v1', + OPENAI_API_KEY: 'sk-openai-should-not-leak', + OPENAI_BASE_URL: 'https://api.openai.com/v1', + }, + }); + const config = JSON.parse(result.configContent) as { + provider: Record }>; + }; + + expect(config.provider['openai-compatible']).toMatchObject({ + npm: '@ai-sdk/openai-compatible', + options: { + baseURL: 'https://proxy.example.com/v1', + }, + models: { 'gpt-4o': { name: 'gpt-4o' } }, + }); + expect( + config.provider['openai-compatible']?.options.apiKey, + ).toBeUndefined(); + }); + + it('rebases gateway-backed openai-compatible providers with the run token', () => { + const result = generateOpenCodeConfig({ + homeDir: createHomeDir(), + runtimeEnv: { + R_MODEL: 'openai-compatible/gpt-4o', + R_INFERENCE_GATEWAY_URL: 'https://api.example.com/api/inference/', + }, + }); + const config = JSON.parse(result.configContent) as { + provider: Record }>; + }; + + expect(config.provider['openai-compatible']?.options).toMatchObject({ + baseURL: 'https://api.example.com/api/inference/openai-compatible/v1', + apiKey: '{env:ROOMOTE_CLOUD_TOKEN}', + }); }); it('rebases gateway-backed compatible providers without an Ollama key', () => { diff --git a/apps/worker/src/run-task/agent-home.ts b/apps/worker/src/run-task/agent-home.ts index ae63d111c..fb0a4a717 100644 --- a/apps/worker/src/run-task/agent-home.ts +++ b/apps/worker/src/run-task/agent-home.ts @@ -70,26 +70,40 @@ export const OPENCODE_AUTH_FILE_NAME = 'auth.json'; const OPENROUTER_PROVIDER_ID = 'openrouter'; const OPENAI_COMPATIBLE_PROVIDER_CONFIGS = { + 'openai-compatible': { + name: 'OpenAI-compatible', + baseUrlEnvVarName: 'OPENAI_COMPATIBLE_BASE_URL', + fallbackBaseUrl: 'http://127.0.0.1:4000/v1', + apiKeyEnvVarName: 'OPENAI_COMPATIBLE_API_KEY' as string | undefined, + keyless: false, + // Arbitrary endpoints must not inherit OPENAI_* credentials. When the + // optional dedicated key is blank, omit the API key entirely so keyless + // servers that reject bearer auth match setup qualification behavior. + allowOpenAiEnvFallback: false, + }, ollama: { name: 'Ollama', baseUrlEnvVarName: 'OLLAMA_BASE_URL', fallbackBaseUrl: 'http://127.0.0.1:11434/v1', - apiKeyEnvVarName: undefined, + apiKeyEnvVarName: undefined as string | undefined, keyless: true, + allowOpenAiEnvFallback: false, }, vllm: { name: 'vLLM', baseUrlEnvVarName: 'VLLM_BASE_URL', fallbackBaseUrl: 'http://127.0.0.1:8000/v1', - apiKeyEnvVarName: 'VLLM_API_KEY', + apiKeyEnvVarName: 'VLLM_API_KEY' as string | undefined, keyless: false, + allowOpenAiEnvFallback: true, }, litellm: { name: 'LiteLLM', baseUrlEnvVarName: 'LITELLM_BASE_URL', fallbackBaseUrl: 'http://127.0.0.1:4000/v1', - apiKeyEnvVarName: 'LITELLM_API_KEY', + apiKeyEnvVarName: 'LITELLM_API_KEY' as string | undefined, keyless: false, + allowOpenAiEnvFallback: true, }, } as const; @@ -706,6 +720,19 @@ function mergeOpenAiCompatibleProviderConfig( const directApiKey = provider.apiKeyEnvVarName ? runtimeEnv[provider.apiKeyEnvVarName]?.trim() : undefined; + const baseURL = + runtimeEnv[provider.baseUrlEnvVarName]?.trim() || + (provider.allowOpenAiEnvFallback + ? runtimeEnv.OPENAI_BASE_URL?.trim() + : '') || + provider.fallbackBaseUrl; + const apiKeyOptions = directApiKey + ? { apiKey: `{env:${provider.apiKeyEnvVarName}}` } + : provider.keyless + ? { apiKey: 'ollama' } + : provider.allowOpenAiEnvFallback && runtimeEnv.OPENAI_API_KEY?.trim() + ? { apiKey: '{env:OPENAI_API_KEY}' } + : {}; merged = { ...merged, @@ -715,19 +742,10 @@ function mergeOpenAiCompatibleProviderConfig( name: provider.name, options: { ...existingOptions, - baseURL: - runtimeEnv[provider.baseUrlEnvVarName]?.trim() || - (!provider.keyless ? runtimeEnv.OPENAI_BASE_URL?.trim() : '') || - provider.fallbackBaseUrl, + baseURL, // OpenAI-compatible clients require an API key even though Ollama // itself accepts unauthenticated requests. - ...(directApiKey - ? { apiKey: `{env:${provider.apiKeyEnvVarName}}` } - : provider.keyless - ? { apiKey: 'ollama' } - : runtimeEnv.OPENAI_API_KEY?.trim() - ? { apiKey: '{env:OPENAI_API_KEY}' } - : {}), + ...apiKeyOptions, }, models: { ...existingModels, diff --git a/packages/types/src/__tests__/inference-gateway.test.ts b/packages/types/src/__tests__/inference-gateway.test.ts index 3989379f7..925962a36 100644 --- a/packages/types/src/__tests__/inference-gateway.test.ts +++ b/packages/types/src/__tests__/inference-gateway.test.ts @@ -101,6 +101,14 @@ describe('inference gateway key lookups', () => { }); it('registers local OpenAI-compatible endpoint providers', () => { + expect(getInferenceGatewayProvider('openai-compatible')).toMatchObject({ + upstreamBaseUrlEnvVarName: 'OPENAI_COMPATIBLE_BASE_URL', + gatewayEnvVarNames: [ + 'OPENAI_COMPATIBLE_BASE_URL', + 'OPENAI_COMPATIBLE_API_KEY', + ], + optionalApiKey: true, + }); expect(getInferenceGatewayProvider('litellm')).toMatchObject({ upstreamBaseUrlEnvVarName: 'LITELLM_BASE_URL', gatewayEnvVarNames: ['LITELLM_BASE_URL', 'LITELLM_API_KEY'], diff --git a/packages/types/src/inference-gateway.ts b/packages/types/src/inference-gateway.ts index fe49f93ab..c8eb090c4 100644 --- a/packages/types/src/inference-gateway.ts +++ b/packages/types/src/inference-gateway.ts @@ -340,6 +340,20 @@ export const INFERENCE_GATEWAY_PROVIDERS: readonly InferenceGatewayProvider[] = allowedPaths: ANTHROPIC_COMPATIBLE_INFERENCE_PATHS, openCodeBaseUrlSuffix: '/v1', }, + { + id: 'openai-compatible', + name: 'OpenAI-compatible', + envVarNames: ['OPENAI_COMPATIBLE_API_KEY'], + gatewayEnvVarNames: [ + 'OPENAI_COMPATIBLE_BASE_URL', + 'OPENAI_COMPATIBLE_API_KEY', + ], + upstreamBaseUrlEnvVarName: 'OPENAI_COMPATIBLE_BASE_URL', + authHeader: { name: 'authorization', scheme: 'bearer' }, + optionalApiKey: true, + allowedPaths: OPENAI_COMPATIBLE_INFERENCE_PATHS, + openCodeBaseUrlSuffix: '/v1', + }, { id: 'litellm', name: 'LiteLLM', diff --git a/packages/types/src/model-provider-config.test.ts b/packages/types/src/model-provider-config.test.ts index 1b2f313eb..6cbb8b026 100644 --- a/packages/types/src/model-provider-config.test.ts +++ b/packages/types/src/model-provider-config.test.ts @@ -231,6 +231,7 @@ describe('SETUP_MODEL_PROVIDER_CATALOG', () => { 'google', 'xai', 'github-copilot', + 'openai-compatible', 'litellm', 'ollama', 'vllm', diff --git a/packages/types/src/model-provider-config.ts b/packages/types/src/model-provider-config.ts index c221e18e7..00ce0c1c2 100644 --- a/packages/types/src/model-provider-config.ts +++ b/packages/types/src/model-provider-config.ts @@ -512,6 +512,24 @@ export const SETUP_MODEL_PROVIDER_CATALOG = [ planning: 'github-copilot/claude-opus-4.8', }, }, + { + id: 'openai-compatible', + label: 'OpenAI-compatible', + envVarName: 'OPENAI_COMPATIBLE_BASE_URL', + envVarLabel: 'Endpoint URL', + additionalEnvFields: [ + { + envVarName: 'OPENAI_COMPATIBLE_API_KEY', + label: 'API key (optional)', + secret: true, + required: false, + }, + ], + defaultRoomoteModel: '', + authKind: 'endpoint', + suggestedTaskModels: [], + dynamicModels: true, + }, { id: 'litellm', label: 'LiteLLM', diff --git a/packages/types/src/task-models.ts b/packages/types/src/task-models.ts index 9173ed7bf..f62111546 100644 --- a/packages/types/src/task-models.ts +++ b/packages/types/src/task-models.ts @@ -30,6 +30,7 @@ export const ENABLED_DIRECT_TASK_MODEL_PROVIDER_IDS = [ 'google', 'xai', 'github-copilot', + 'openai-compatible', 'litellm', 'ollama', 'vllm',