From 416af553eeda7436623d2ea815fa24f24c7faa92 Mon Sep 17 00:00:00 2001 From: Aswin Karumbunathan Date: Mon, 16 Mar 2026 15:42:22 -0700 Subject: [PATCH 1/2] sync_models: Handle model name collisions This is particularly important for the available_providers list, where we want to list all of the matching providers if there are multiple. For example, "gemini-flash" from vertex and "gemini/gemini-flash" would be collapsed into a single entry, but we want both providers in available_providers. This doesn't handle different pricing or parameters for the same model on different providers, but is at least more deterministic than previously. For now we always choose the provider that is alphabetically first. --- packages/proxy/schema/model_list.json | 47 ++++-- packages/proxy/scripts/sync_models.ts | 233 +++++++++++++------------- 2 files changed, 149 insertions(+), 131 deletions(-) diff --git a/packages/proxy/schema/model_list.json b/packages/proxy/schema/model_list.json index 749ad12e..a8f6eb0e 100644 --- a/packages/proxy/schema/model_list.json +++ b/packages/proxy/schema/model_list.json @@ -1492,7 +1492,8 @@ "max_input_tokens": 131072, "max_output_tokens": 32766, "available_providers": [ - "groq" + "groq", + "together" ] }, "openai/gpt-oss-20b": { @@ -1506,7 +1507,8 @@ "max_input_tokens": 131072, "max_output_tokens": 32768, "available_providers": [ - "groq" + "groq", + "together" ] }, "accounts/fireworks/models/gpt-oss-120b": { @@ -2924,6 +2926,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65536, "available_providers": [ + "google", "vertex" ] }, @@ -2940,6 +2943,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65536, "available_providers": [ + "google", "vertex" ] }, @@ -2956,6 +2960,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65536, "available_providers": [ + "google", "vertex" ] }, @@ -2969,10 +2974,11 @@ "displayName": "Gemini 3 Pro (Preview)", "reasoning": true, "reasoning_budget": true, - "deprecation_date": "2026-03-26", + "deprecation_date": "2026-03-09", "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -2989,7 +2995,8 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ - "google" + "google", + "vertex" ] }, "gemini-2.5-flash": { @@ -3005,6 +3012,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3021,6 +3029,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3038,6 +3047,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3164,6 +3174,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3183,6 +3194,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3199,6 +3211,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 65535, "available_providers": [ + "google", "vertex" ] }, @@ -3214,6 +3227,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "available_providers": [ + "google", "vertex" ] }, @@ -3221,14 +3235,15 @@ "format": "google", "flavor": "chat", "multimodal": true, - "input_cost_per_mil_tokens": 0.15, - "output_cost_per_mil_tokens": 0.6, - "input_cache_read_cost_per_mil_tokens": 0.0375, + "input_cost_per_mil_tokens": 0.1, + "output_cost_per_mil_tokens": 0.4, + "input_cache_read_cost_per_mil_tokens": 0.025, "deprecation_date": "2026-06-01", "parent": "gemini-2.0-flash", "max_input_tokens": 1048576, "max_output_tokens": 8192, "available_providers": [ + "google", "vertex" ] }, @@ -3244,6 +3259,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "available_providers": [ + "google", "vertex" ] }, @@ -3259,6 +3275,7 @@ "max_input_tokens": 1048576, "max_output_tokens": 8192, "available_providers": [ + "google", "vertex" ] }, @@ -3272,6 +3289,7 @@ "max_input_tokens": 65536, "max_output_tokens": 32768, "available_providers": [ + "google", "vertex" ] }, @@ -3280,13 +3298,13 @@ "flavor": "chat", "multimodal": true, "input_cost_per_mil_tokens": 0.075, - "output_cost_per_mil_tokens": 0.3, + "output_cost_per_mil_tokens": 0, "displayName": "Gemini 1.5 Flash", "deprecation_date": "2025-09-29", - "max_input_tokens": 1000000, + "max_input_tokens": 8192, "max_output_tokens": 8192, "available_providers": [ - "vertex" + "google" ] }, "gemini-1.5-flash-latest": { @@ -3488,12 +3506,13 @@ "format": "google", "flavor": "chat", "multimodal": true, - "input_cost_per_mil_tokens": 0, - "output_cost_per_mil_tokens": 0, + "input_cost_per_mil_tokens": 0.3, + "output_cost_per_mil_tokens": 2.5, + "input_cache_read_cost_per_mil_tokens": 0.03, "experimental": true, "deprecated": true, - "max_input_tokens": 2097152, - "max_output_tokens": 8192, + "max_input_tokens": 1048576, + "max_output_tokens": 65535, "available_providers": [ "google" ] diff --git a/packages/proxy/scripts/sync_models.ts b/packages/proxy/scripts/sync_models.ts index 07918215..cd9451ff 100644 --- a/packages/proxy/scripts/sync_models.ts +++ b/packages/proxy/scripts/sync_models.ts @@ -282,6 +282,78 @@ function getProviderMappingForModel( return result; } +type ResolvedRemoteEntry = { + remoteModelName: string; + remoteModel: LiteLLMModelDetail; + mergedProviders: string[]; +}; + +// Deduplicate remote models by their translated (local) name. +// When multiple remote names translate to the same local name, the model data +// from the first entry is kept and providers from all entries are merged. +function resolveRemoteModels( + remoteModels: LiteLLMModelList, + providerFilter?: string, +): Map { + const result = new Map(); + + // Sort by provider then model name so collision resolution is deterministic + // regardless of JSON key order. The alphabetically earliest provider wins as + // the primary entry (its model data is kept); remaining entries only contribute + // their providers to the merged list. + const sortedNames = Object.keys(remoteModels).sort((a, b) => { + const pa = remoteModels[a].litellm_provider ?? ""; + const pb = remoteModels[b].litellm_provider ?? ""; + return pa !== pb ? pa.localeCompare(pb) : a.localeCompare(b); + }); + + for (const remoteModelName of sortedNames) { + const remoteModel = remoteModels[remoteModelName]; + + if (providerFilter) { + const lowerFilter = providerFilter.toLowerCase(); + const modelProvider = remoteModel.litellm_provider?.toLowerCase(); + const modelNamePart = remoteModelName.split("/")[0].toLowerCase(); + if ( + !modelProvider?.includes(lowerFilter) && + !modelNamePart.includes(lowerFilter) && + modelProvider !== lowerFilter && + modelNamePart !== lowerFilter + ) { + continue; + } + } + + const translatedName = translateToBraintrust( + remoteModelName, + remoteModel.litellm_provider, + ); + const providers = getProviderMappingForModel(remoteModelName, remoteModel); + + if (result.has(translatedName)) { + const existing = result.get(translatedName)!; + const newProviders = providers.filter( + (p) => !existing.mergedProviders.includes(p), + ); + const mergedProviders = [...existing.mergedProviders, ...newProviders]; + if (newProviders.length > 0) { + console.warn( + `⚠️ Collision: "${remoteModelName}" and "${existing.remoteModelName}" both translate to "${translatedName}" — merging providers: ${JSON.stringify(mergedProviders)}`, + ); + } else { + console.warn( + `⚠️ Collision: "${remoteModelName}" and "${existing.remoteModelName}" both translate to "${translatedName}" — same providers, keeping first entry`, + ); + } + result.set(translatedName, { ...existing, mergedProviders }); + } else { + result.set(translatedName, { remoteModelName, remoteModel, mergedProviders: providers }); + } + } + + return result; +} + async function updateProviderMapping( newModels: Array<{ name: string; @@ -678,46 +750,19 @@ async function findMissingCommand(argv: any) { const missingInLocal: string[] = []; const consideredRemoteModels: LiteLLMModelList = {}; - for (const remoteModelName in remoteModels) { - const modelDetail = remoteModels[remoteModelName]; + const resolvedRemote = resolveRemoteModels(remoteModels, argv.provider); - if (argv.provider) { - const lowerArgProvider = argv.provider.toLowerCase(); - const modelProvider = modelDetail.litellm_provider?.toLowerCase(); - const modelNameProviderPart = remoteModelName - .split("/")[0] - .toLowerCase(); - - if ( - !modelProvider?.includes(lowerArgProvider) && - !modelNameProviderPart.includes(lowerArgProvider) && - !(modelProvider === lowerArgProvider) && - !(modelNameProviderPart === lowerArgProvider) - ) { - continue; - } - } - consideredRemoteModels[remoteModelName] = modelDetail; - } - - const remoteModelNamesFiltered = new Set( - Object.keys(consideredRemoteModels), - ); - - for (const modelName of remoteModelNamesFiltered) { - const translatedModelName = translateToBraintrust( - modelName, - consideredRemoteModels[modelName]?.litellm_provider, - ); + for (const [translatedName, { remoteModelName, remoteModel }] of resolvedRemote) { + consideredRemoteModels[remoteModelName] = remoteModel; if (argv.provider) { console.log( - `[DEBUG] Remote: ${modelName} (Provider: ${ - consideredRemoteModels[modelName]?.litellm_provider || "N/A" - }) -> Translated: ${translatedModelName}`, + `[DEBUG] Remote: ${remoteModelName} (Provider: ${ + remoteModel.litellm_provider || "N/A" + }) -> Translated: ${translatedName}`, ); } - if (!localModelNames.has(translatedModelName)) { - missingInLocal.push(modelName); + if (!localModelNames.has(translatedName)) { + missingInLocal.push(remoteModelName); } } @@ -886,61 +931,34 @@ async function updateModelsCommand(argv: any) { localModelDetail: LocalModelDetail; remoteModelName: string; remoteModelDetail: LiteLLMModelDetail; + mergedProviders: string[]; }> = []; + const resolvedRemote = resolveRemoteModels(remoteModels, argv.provider); + if (argv.provider) { - const lowerArgProvider = argv.provider.toLowerCase(); - for (const remoteModelName in remoteModels) { - const remoteModelDetail = remoteModels[remoteModelName]; - const modelProvider = remoteModelDetail.litellm_provider?.toLowerCase(); - const modelNameProviderPart = remoteModelName - .split("/")[0] - .toLowerCase(); - - const matchesProviderFilter = - modelProvider?.includes(lowerArgProvider) || - modelNameProviderPart.includes(lowerArgProvider) || - modelProvider === lowerArgProvider || - modelNameProviderPart === lowerArgProvider; - - if (matchesProviderFilter) { - const translatedRemoteModelName = translateToBraintrust( + for (const [translatedRemoteModelName, { remoteModelName, remoteModel: remoteModelDetail, mergedProviders }] of resolvedRemote) { + if (localModels[translatedRemoteModelName]) { + modelsToCompare.push({ + localModelName: translatedRemoteModelName, + localModelDetail: localModels[translatedRemoteModelName], remoteModelName, - remoteModelDetail.litellm_provider, - ); - if (localModels[translatedRemoteModelName]) { - modelsToCompare.push({ - localModelName: translatedRemoteModelName, - localModelDetail: localModels[translatedRemoteModelName], - remoteModelName: remoteModelName, - remoteModelDetail: remoteModelDetail, - }); - } + remoteModelDetail, + mergedProviders, + }); } } } else { for (const localModelName in localModels) { const localModelDetail = localModels[localModelName]; - let foundRemoteDetail: LiteLLMModelDetail | undefined = undefined; - let originalRemoteModelNameForLoop: string | undefined = undefined; - for (const rName in remoteModels) { - const rDetail = remoteModels[rName]; - const translatedName = translateToBraintrust( - rName, - rDetail.litellm_provider, - ); - if (translatedName === localModelName) { - foundRemoteDetail = rDetail; - originalRemoteModelNameForLoop = rName; - break; - } - } - if (foundRemoteDetail && originalRemoteModelNameForLoop) { + const resolvedEntry = resolvedRemote.get(localModelName); + if (resolvedEntry) { modelsToCompare.push({ - localModelName: localModelName, - localModelDetail: localModelDetail, - remoteModelName: originalRemoteModelNameForLoop, - remoteModelDetail: foundRemoteDetail, + localModelName, + localModelDetail, + remoteModelName: resolvedEntry.remoteModelName, + remoteModelDetail: resolvedEntry.remoteModel, + mergedProviders: resolvedEntry.mergedProviders, }); } } @@ -952,6 +970,7 @@ async function updateModelsCommand(argv: any) { localModelDetail, remoteModelName: originalRemoteModelName, remoteModelDetail, + mergedProviders, } = item; const modelInUpdatedList = updatedLocalModels[localModelName]; @@ -1189,11 +1208,8 @@ async function updateModelsCommand(argv: any) { remoteDeprecationDate, ); - // Set available_providers from remote - const remoteProviders = getProviderMappingForModel( - originalRemoteModelName, - remoteModelDetail, - ); + // Set available_providers from remote (using merged providers across all colliding remote entries) + const remoteProviders = mergedProviders; if (remoteProviders.length > 0) { const currentProviders = (modelInUpdatedList as any) .available_providers; @@ -1294,34 +1310,12 @@ async function addModelsCommand(argv: any) { remoteModelName: string; translatedName: string; remoteModel: LiteLLMModelDetail; + mergedProviders: string[]; }> = []; - // Find missing models - for (const remoteModelName in remoteModels) { - const modelDetail = remoteModels[remoteModelName]; - - if (argv.provider) { - const lowerArgProvider = argv.provider.toLowerCase(); - const modelProvider = modelDetail.litellm_provider?.toLowerCase(); - const modelNameProviderPart = remoteModelName - .split("/")[0] - .toLowerCase(); - - if ( - !modelProvider?.includes(lowerArgProvider) && - !modelNameProviderPart.includes(lowerArgProvider) && - !(modelProvider === lowerArgProvider) && - !(modelNameProviderPart === lowerArgProvider) - ) { - continue; - } - } - - const translatedModelName = translateToBraintrust( - remoteModelName, - modelDetail.litellm_provider, - ); - + // Find missing models, deduplicating by translated name and merging providers + const resolvedRemote = resolveRemoteModels(remoteModels, argv.provider); + for (const [translatedModelName, { remoteModelName, remoteModel: modelDetail, mergedProviders }] of resolvedRemote) { if (argv.filter) { const lowerFilter = argv.filter.toLowerCase(); if ( @@ -1337,6 +1331,7 @@ async function addModelsCommand(argv: any) { remoteModelName, translatedName: translatedModelName, remoteModel: modelDetail, + mergedProviders, }); } } @@ -1388,19 +1383,23 @@ async function addModelsCommand(argv: any) { // Convert remote models to local format const modelsToAdd = missingInLocal.map( - ({ remoteModelName, translatedName, remoteModel }) => ({ - name: translatedName, - model: convertRemoteToLocalModel(remoteModelName, remoteModel), - }), + ({ remoteModelName, translatedName, remoteModel, mergedProviders }) => { + const model = convertRemoteToLocalModel(remoteModelName, remoteModel); + // Override with merged providers (may include providers from colliding remote entries) + if (mergedProviders.length > 0) { + model.available_providers = mergedProviders as ModelSpec["available_providers"]; + } + return { name: translatedName, model }; + }, ); const newModelNames = modelsToAdd.map((m) => m.name); // Prepare provider mapping data const providerMappingData = missingInLocal.map( - ({ translatedName, remoteModel }) => ({ + ({ translatedName, remoteModel, mergedProviders }) => ({ name: translatedName, - providers: getProviderMappingForModel(translatedName, remoteModel), + providers: mergedProviders, remoteModel: remoteModel, }), ); From b2440515fd61946a0a3da3026c76b1a73f50682e Mon Sep 17 00:00:00 2001 From: Aswin Karumbunathan Date: Mon, 16 Mar 2026 16:24:10 -0700 Subject: [PATCH 2/2] Fix codex comment --- packages/proxy/scripts/sync_models.ts | 48 ++++++++++++++++++--------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/packages/proxy/scripts/sync_models.ts b/packages/proxy/scripts/sync_models.ts index cd9451ff..1e5d435b 100644 --- a/packages/proxy/scripts/sync_models.ts +++ b/packages/proxy/scripts/sync_models.ts @@ -191,6 +191,27 @@ function translateToBraintrust(modelName: string, provider?: string): string { return modelName; } +function matchesProviderFilter( + remoteModelName: string, + remoteModel: LiteLLMModelDetail, + providerFilter?: string, +): boolean { + if (!providerFilter) { + return true; + } + + const lowerFilter = providerFilter.toLowerCase(); + const modelProvider = remoteModel.litellm_provider?.toLowerCase(); + const modelNamePart = remoteModelName.split("/")[0].toLowerCase(); + + return ( + modelProvider?.includes(lowerFilter) || + modelNamePart.includes(lowerFilter) || + modelProvider === lowerFilter || + modelNamePart === lowerFilter + ); +} + function getProviderMappingForModel( remoteModelName: string, remoteModel: LiteLLMModelDetail, @@ -310,18 +331,8 @@ function resolveRemoteModels( for (const remoteModelName of sortedNames) { const remoteModel = remoteModels[remoteModelName]; - if (providerFilter) { - const lowerFilter = providerFilter.toLowerCase(); - const modelProvider = remoteModel.litellm_provider?.toLowerCase(); - const modelNamePart = remoteModelName.split("/")[0].toLowerCase(); - if ( - !modelProvider?.includes(lowerFilter) && - !modelNamePart.includes(lowerFilter) && - modelProvider !== lowerFilter && - modelNamePart !== lowerFilter - ) { - continue; - } + if (!matchesProviderFilter(remoteModelName, remoteModel, providerFilter)) { + continue; } const translatedName = translateToBraintrust( @@ -749,8 +760,15 @@ async function findMissingCommand(argv: any) { const localModelNames = new Set(Object.keys(localModels)); const missingInLocal: string[] = []; const consideredRemoteModels: LiteLLMModelList = {}; + const filteredRemoteModels: LiteLLMModelList = {}; - const resolvedRemote = resolveRemoteModels(remoteModels, argv.provider); + for (const [remoteModelName, remoteModel] of Object.entries(remoteModels)) { + if (matchesProviderFilter(remoteModelName, remoteModel, argv.provider)) { + filteredRemoteModels[remoteModelName] = remoteModel; + } + } + + const resolvedRemote = resolveRemoteModels(filteredRemoteModels); for (const [translatedName, { remoteModelName, remoteModel }] of resolvedRemote) { consideredRemoteModels[remoteModelName] = remoteModel; @@ -775,8 +793,8 @@ async function findMissingCommand(argv: any) { [provider: string]: { totalRemote: number; missingInLocal: number }; } = {}; - for (const modelName in consideredRemoteModels) { - const modelDetail = consideredRemoteModels[modelName]; + for (const modelName in filteredRemoteModels) { + const modelDetail = filteredRemoteModels[modelName]; const provider = modelDetail.litellm_provider || "Unknown Provider"; if (!providerSummary[provider]) { providerSummary[provider] = { totalRemote: 0, missingInLocal: 0 };