From b5b7c39127db5cb922688dce5b47b87f20175ab7 Mon Sep 17 00:00:00 2001 From: Professor Synapse <131487882+ProfSynapse@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:34:36 -0400 Subject: [PATCH] fix(providers): test keys with the cheapest registry models, never a hardcoded one Perplexity and Requesty cannot check a key without sending a message. They now take the cheapest models from their own registry (priced models first, up to three) and move to the next when one is refused as a model; a refusal of the key stops at once. Requesty's old test model, openai/gpt-5-nano, was not even in its registry. A guard test fails if ValidationService names a model id again. Remove Gemini 2.5 Flash and Gemini 2.5 Pro from Requesty: Google retires both (Cloud 2026-10-20) and new accounts already get "no longer available". Co-Authored-By: Claude Opus 5.5 --- docs/changelog.md | 2 + .../llm/adapters/requesty/RequestyModels.ts | 32 ------- .../llm/validation/ValidationService.ts | 89 +++++++++++++------ tests/unit/LLMValidationService.test.ts | 72 +++++++++++++-- 4 files changed, 127 insertions(+), 68 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 742b917a..dca4afd8 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -8,6 +8,8 @@ - Adding an OpenRouter API key failed with "No endpoints found for liquid/lfm-2.5-1.2b-thinking:free", so OpenRouter stayed turned off even with a working key. The check sent a test message to a free model that OpenRouter has since removed. It now asks OpenRouter whether the key is valid, which costs nothing and does not depend on any model. - No provider's key check can be failed by a retired model any more. OpenAI, Anthropic, Google, Mistral, Groq and DeepSeek keys are checked by listing models instead of sending a test message, which also fixes Groq keys, whose test model had been retired. Perplexity and Requesty still send a test message, but an error about the test model no longer counts as a bad key. An OpenAI key restricted from listing models is accepted. +- Perplexity and Requesty, which have no way to check a key without a message, now test with the cheapest models in Nexus's own list for that provider and move on to the next cheapest if one has been retired, instead of a fixed model that could disappear. +- Gemini 2.5 Flash and Gemini 2.5 Pro are removed from Requesty. Google is retiring both, and new accounts already cannot use them. - Provider settings mark models the provider no longer offers as **No longer available** and lock their toggle off, instead of letting you turn on a model that will fail. Your saved choice is kept in case the model returns. If Nexus cannot reach the model list, every model stays available. --- diff --git a/src/services/llm/adapters/requesty/RequestyModels.ts b/src/services/llm/adapters/requesty/RequestyModels.ts index 47595c3c..4d041a24 100644 --- a/src/services/llm/adapters/requesty/RequestyModels.ts +++ b/src/services/llm/adapters/requesty/RequestyModels.ts @@ -138,38 +138,6 @@ export const REQUESTY_MODELS: ModelSpec[] = [ supportsThinking: true } }, - { - provider: 'requesty', - name: 'Gemini 2.5 Pro', - apiName: 'google/gemini-2.5-pro', - contextWindow: 1048576, - maxTokens: 66000, - inputCostPerMillion: 1.25, - outputCostPerMillion: 10.00, - capabilities: { - supportsJSON: true, - supportsImages: true, - supportsFunctions: true, - supportsStreaming: true, - supportsThinking: true - } - }, - { - provider: 'requesty', - name: 'Gemini 2.5 Flash', - apiName: 'google/gemini-2.5-flash', - contextWindow: 1048576, - maxTokens: 65536, - inputCostPerMillion: 0.15, - outputCostPerMillion: 0.60, - capabilities: { - supportsJSON: true, - supportsImages: true, - supportsFunctions: true, - supportsStreaming: true, - supportsThinking: true - } - }, // Anthropic models via Requesty (dashed upstream slugs) // Requesty lists this one with a dotted version, unlike its dashed Claude siblings. diff --git a/src/services/llm/validation/ValidationService.ts b/src/services/llm/validation/ValidationService.ts index 595a0615..87ce81b3 100644 --- a/src/services/llm/validation/ValidationService.ts +++ b/src/services/llm/validation/ValidationService.ts @@ -13,6 +13,9 @@ import { ProviderHttpClient, ProviderHttpResponse } from '../adapters/shared/ProviderHttpClient'; +import type { ModelSpec } from '../adapters/modelTypes'; +import { PERPLEXITY_MODELS } from '../adapters/perplexity/PerplexityModels'; +import { REQUESTY_MODELS } from '../adapters/requesty/RequestyModels'; type ValidationResponseBody = { error?: { @@ -124,36 +127,69 @@ export class LLMValidationService { } } + /** How many registry models a test-message check tries before giving up. */ + private static readonly TEST_MODEL_ATTEMPTS = 3; + + /** + * The provider's own registry models, cheapest first, for a test message. + * Taking them from the registry instead of a hardcoded id means the check + * moves with the model list, and a model with no price (unknown or free, + * which is how a removed free model broke the OpenRouter check) goes last. + */ + static cheapestTestModels(models: readonly ModelSpec[]): string[] { + const cost = (model: ModelSpec) => model.inputCostPerMillion + model.outputCostPerMillion; + return models + .map((model, index) => ({ model, index })) + .sort((a, b) => { + const aPriced = cost(a.model) > 0; + const bPriced = cost(b.model) > 0; + if (aPriced !== bPriced) return aPriced ? -1 : 1; + return cost(a.model) - cost(b.model) || a.index - b.index; + }) + .map(({ model }) => model.apiName) + .filter((id, index, ids) => ids.indexOf(id) === index) + .slice(0, this.TEST_MODEL_ATTEMPTS); + } + /** * Validate a key by sending a one-word test message, for providers that - * cannot check a key any other way. An error about the test model itself - * means the key was accepted, so a retired model never fails the provider. + * cannot check a key any other way. It tries the cheapest registry models in + * turn: an error about the model moves on to the next one, and an error + * about the key stops. If every model was refused only as a model, the key + * itself was accepted, so a retired model never fails the provider. */ private static async validateWithTestMessage( provider: string, label: string, url: string, headers: Record, - body: Record + models: string[], + buildBody: (model: string) => Record ): Promise<{ success: boolean; error?: string }> { + if (models.length === 0) { + return { success: false, error: `No ${label} model is available to check the key with` }; + } + try { - const response = await this.requestWithTimeout(provider, `${label} validation`, { - url, - method: 'POST', - headers: { 'Content-Type': 'application/json', ...headers }, - body: JSON.stringify(body) - }); + for (const model of models) { + const response = await this.requestWithTimeout(provider, `${label} validation`, { + url, + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(buildBody(model)) + }); - if (response.status >= 200 && response.status < 300) { - return { success: true }; - } + if (response.status >= 200 && response.status < 300) { + return { success: true }; + } - const errorData = (response.json ?? {}) as ValidationResponseBody & { message?: string; detail?: string }; - const error = errorData.error?.message || errorData.message || errorData.detail || `HTTP ${response.status}`; - if (this.isTestModelUnavailable(response.status, error)) { - return { success: true }; + const errorData = (response.json ?? {}) as ValidationResponseBody & { message?: string; detail?: string }; + const error = errorData.error?.message || errorData.message || errorData.detail || `HTTP ${response.status}`; + if (!this.isTestModelUnavailable(response.status, error)) { + return { success: false, error }; + } } - return { success: false, error }; + return { success: true }; } catch (error: unknown) { return { success: false, @@ -349,28 +385,27 @@ export class LLMValidationService { } private static async validatePerplexity(apiKey: string): Promise<{ success: boolean; error?: string }> { - // Perplexity has no model list, so this sends a test message. A retired - // test model does not fail the key. + // Perplexity has no model list, so this sends a test message to its + // cheapest registry models. return this.validateWithTestMessage('perplexity', 'Perplexity', 'https://api.perplexity.ai/chat/completions', { 'Authorization': `Bearer ${apiKey}`, 'X-Pplx-Integration': 'nexus' - }, { - model: 'sonar', + }, this.cheapestTestModels(PERPLEXITY_MODELS), model => ({ + model, messages: [{ role: 'user', content: 'Hi' }], max_tokens: 5 - }); + })); } private static async validateRequesty(apiKey: string): Promise<{ success: boolean; error?: string }> { // Requesty answers a bad key on its model list with the same 403 as other - // refusals, so this sends a test message. A retired test model does not - // fail the key. + // refusals, so this sends a test message to its cheapest registry models. return this.validateWithTestMessage('requesty', 'Requesty', 'https://router.requesty.ai/v1/chat/completions', { 'Authorization': `Bearer ${apiKey}` - }, { - model: 'openai/gpt-5-nano', + }, this.cheapestTestModels(REQUESTY_MODELS), model => ({ + model, messages: [{ role: 'user', content: 'Hi' }], max_completion_tokens: 5 - }); + })); } } diff --git a/tests/unit/LLMValidationService.test.ts b/tests/unit/LLMValidationService.test.ts index 458607d9..d1da2695 100644 --- a/tests/unit/LLMValidationService.test.ts +++ b/tests/unit/LLMValidationService.test.ts @@ -7,7 +7,12 @@ */ import { __setRequestUrlMock } from 'obsidian'; +import { readFileSync } from 'fs'; +import { join } from 'path'; import { LLMValidationService } from '../../src/services/llm/validation/ValidationService'; +import type { ModelSpec } from '../../src/services/llm/adapters/modelTypes'; +import { PERPLEXITY_MODELS } from '../../src/services/llm/adapters/perplexity/PerplexityModels'; +import { REQUESTY_MODELS } from '../../src/services/llm/adapters/requesty/RequestyModels'; const mockRequestUrl = jest.fn(); @@ -84,19 +89,44 @@ describe('LLMValidationService', () => { await expect(validator('validateOpenAI')('restricted-key')).resolves.toEqual({ success: true }); }); - describe.each([['validatePerplexity'], ['validateRequesty']])('%s', (name) => { - it('accepts the key when only the test model is unavailable', async () => { - mockRequestUrl.mockResolvedValue({ - status: 404, - json: { error: { message: 'No endpoints found for openai/gpt-5-nano.' } }, - text: '', - headers: {} - }); + describe.each([ + ['validatePerplexity', PERPLEXITY_MODELS], + ['validateRequesty', REQUESTY_MODELS] + ] as const)('%s', (name, registry) => { + const expected = LLMValidationService.cheapestTestModels(registry); + const sentModels = () => mockRequestUrl.mock.calls.map(call => JSON.parse(call[0].body).model); + const modelGone = (model: string) => ({ + status: 404, + json: { error: { message: `No endpoints found for ${model}.` } }, + text: '', + headers: {} + }); + + it('tests with the cheapest model in its own registry', async () => { + mockRequestUrl.mockResolvedValue({ status: 200, json: {}, text: '{}', headers: {} }); + + await expect(validator(name)('test-key')).resolves.toEqual({ success: true }); + expect(sentModels()).toEqual([expected[0]]); + expect(registry.map(model => model.apiName)).toContain(expected[0]); + }); + + it('moves on to the next cheapest model when one is gone', async () => { + mockRequestUrl + .mockResolvedValueOnce(modelGone(expected[0])) + .mockResolvedValueOnce({ status: 200, json: {}, text: '{}', headers: {} }); + + await expect(validator(name)('test-key')).resolves.toEqual({ success: true }); + expect(sentModels()).toEqual(expected.slice(0, 2)); + }); + + it('accepts the key when every test model is gone, because only the models were refused', async () => { + mockRequestUrl.mockImplementation(async (request: { body: string }) => modelGone(JSON.parse(request.body).model)); await expect(validator(name)('test-key')).resolves.toEqual({ success: true }); + expect(sentModels()).toEqual(expected); }); - it('rejects a key the provider refuses', async () => { + it('stops at the first refusal of the key', async () => { mockRequestUrl.mockResolvedValue({ status: 401, json: { error: { message: 'Invalid model access token' } }, @@ -105,6 +135,7 @@ describe('LLMValidationService', () => { }); await expect(validator(name)('bad-key')).resolves.toEqual({ success: false, error: 'Invalid model access token' }); + expect(mockRequestUrl).toHaveBeenCalledTimes(1); }); it('rejects a 400 that blames the key even when it mentions a model', async () => { @@ -118,4 +149,27 @@ describe('LLMValidationService', () => { await expect(validator(name)('bad-key')).resolves.toEqual({ success: false, error: 'Invalid API key for model sonar' }); }); }); + + describe('cheapestTestModels', () => { + const spec = (apiName: string, input: number, output: number) => + ({ apiName, inputCostPerMillion: input, outputCostPerMillion: output }) as ModelSpec; + + it('orders by price, puts unpriced models last, drops duplicates and caps the attempts', () => { + expect(LLMValidationService.cheapestTestModels([ + spec('free', 0, 0), + spec('pricey', 5, 25), + spec('cheap', 0.1, 0.4), + spec('cheap', 0.1, 0.4), + spec('middle', 1, 4) + ])).toEqual(['cheap', 'middle', 'pricey']); + }); + }); + + it('never names a model in a key check: test models come from the registry', () => { + // A hardcoded test model is how the OpenRouter check broke when its free + // model was removed. Checks must be model-free or use cheapestTestModels. + const source = readFileSync(join(__dirname, '../../src/services/llm/validation/ValidationService.ts'), 'utf8'); + expect(source).not.toMatch(/model\s*:\s*['"`]/); + expect(source).not.toMatch(/\/models\/[a-z0-9.-]+:generateContent/i); + }); });