Test API keys with the cheapest registry models; remove Gemini 2.5 from Requesty

docs/changelog.md

@@ -8,6 +8,8 @@
 
 - Adding an OpenRouter API key failed with "No endpoints found for liquid/lfm-2.5-1.2b-thinking:free", so OpenRouter stayed turned off even with a working key. The check sent a test message to a free model that OpenRouter has since removed. It now asks OpenRouter whether the key is valid, which costs nothing and does not depend on any model.
 - No provider's key check can be failed by a retired model any more. OpenAI, Anthropic, Google, Mistral, Groq and DeepSeek keys are checked by listing models instead of sending a test message, which also fixes Groq keys, whose test model had been retired. Perplexity and Requesty still send a test message, but an error about the test model no longer counts as a bad key. An OpenAI key restricted from listing models is accepted.
+- Perplexity and Requesty, which have no way to check a key without a message, now test with the cheapest models in Nexus's own list for that provider and move on to the next cheapest if one has been retired, instead of a fixed model that could disappear.
+- Gemini 2.5 Flash and Gemini 2.5 Pro are removed from Requesty. Google is retiring both, and new accounts already cannot use them.
 - Provider settings mark models the provider no longer offers as **No longer available** and lock their toggle off, instead of letting you turn on a model that will fail. Your saved choice is kept in case the model returns. If Nexus cannot reach the model list, every model stays available.
 ---
 

src/services/llm/adapters/requesty/RequestyModels.ts

@@ -138,38 +138,6 @@ export const REQUESTY_MODELS: ModelSpec[] = [
       supportsThinking: true
     }
   },
-  {
-    provider: 'requesty',
-    name: 'Gemini 2.5 Pro',
-    apiName: 'google/gemini-2.5-pro',
-    contextWindow: 1048576,
-    maxTokens: 66000,
-    inputCostPerMillion: 1.25,
-    outputCostPerMillion: 10.00,
-    capabilities: {
-      supportsJSON: true,
-      supportsImages: true,
-      supportsFunctions: true,
-      supportsStreaming: true,
-      supportsThinking: true
-    }
-  },
-  {
-    provider: 'requesty',
-    name: 'Gemini 2.5 Flash',
-    apiName: 'google/gemini-2.5-flash',
-    contextWindow: 1048576,
-    maxTokens: 65536,
-    inputCostPerMillion: 0.15,
-    outputCostPerMillion: 0.60,
-    capabilities: {
-      supportsJSON: true,
-      supportsImages: true,
-      supportsFunctions: true,
-      supportsStreaming: true,
-      supportsThinking: true
-    }
-  },
 
   // Anthropic models via Requesty (dashed upstream slugs)
   // Requesty lists this one with a dotted version, unlike its dashed Claude siblings.

src/services/llm/validation/ValidationService.ts

@@ -13,6 +13,9 @@ import {
   ProviderHttpClient,
   ProviderHttpResponse
 } from '../adapters/shared/ProviderHttpClient';
+import type { ModelSpec } from '../adapters/modelTypes';
+import { PERPLEXITY_MODELS } from '../adapters/perplexity/PerplexityModels';
+import { REQUESTY_MODELS } from '../adapters/requesty/RequestyModels';
 
 type ValidationResponseBody = {
   error?: {
@@ -124,36 +127,69 @@ export class LLMValidationService {
     }
   }
 
+  /** How many registry models a test-message check tries before giving up. */
+  private static readonly TEST_MODEL_ATTEMPTS = 3;
+
+  /**
+   * The provider's own registry models, cheapest first, for a test message.
+   * Taking them from the registry instead of a hardcoded id means the check
+   * moves with the model list, and a model with no price (unknown or free,
+   * which is how a removed free model broke the OpenRouter check) goes last.
+   */
+  static cheapestTestModels(models: readonly ModelSpec[]): string[] {
+    const cost = (model: ModelSpec) => model.inputCostPerMillion + model.outputCostPerMillion;
+    return models
+      .map((model, index) => ({ model, index }))
+      .sort((a, b) => {
+        const aPriced = cost(a.model) > 0;
+        const bPriced = cost(b.model) > 0;
+        if (aPriced !== bPriced) return aPriced ? -1 : 1;
+        return cost(a.model) - cost(b.model) || a.index - b.index;
+      })
+      .map(({ model }) => model.apiName)
+      .filter((id, index, ids) => ids.indexOf(id) === index)
+      .slice(0, this.TEST_MODEL_ATTEMPTS);
+  }
+
   /**
    * Validate a key by sending a one-word test message, for providers that
-   * cannot check a key any other way. An error about the test model itself
-   * means the key was accepted, so a retired model never fails the provider.
+   * cannot check a key any other way. It tries the cheapest registry models in
+   * turn: an error about the model moves on to the next one, and an error
+   * about the key stops. If every model was refused only as a model, the key
+   * itself was accepted, so a retired model never fails the provider.
    */
   private static async validateWithTestMessage(
     provider: string,
     label: string,
     url: string,
     headers: Record<string, string>,
-    body: Record<string, unknown>
+    models: string[],
+    buildBody: (model: string) => Record<string, unknown>
   ): Promise<{ success: boolean; error?: string }> {
+    if (models.length === 0) {
+      return { success: false, error: `No ${label} model is available to check the key with` };
+    }
+
     try {
-      const response = await this.requestWithTimeout(provider, `${label} validation`, {
-        url,
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json', ...headers },
-        body: JSON.stringify(body)
-      });
+      for (const model of models) {
+        const response = await this.requestWithTimeout(provider, `${label} validation`, {
+          url,
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json', ...headers },
+          body: JSON.stringify(buildBody(model))
+        });
 
-      if (response.status >= 200 && response.status < 300) {
-        return { success: true };
-      }
+        if (response.status >= 200 && response.status < 300) {
+          return { success: true };
+        }
 
-      const errorData = (response.json ?? {}) as ValidationResponseBody & { message?: string; detail?: string };
-      const error = errorData.error?.message || errorData.message || errorData.detail || `HTTP ${response.status}`;
-      if (this.isTestModelUnavailable(response.status, error)) {
-        return { success: true };
+        const errorData = (response.json ?? {}) as ValidationResponseBody & { message?: string; detail?: string };
+        const error = errorData.error?.message || errorData.message || errorData.detail || `HTTP ${response.status}`;
+        if (!this.isTestModelUnavailable(response.status, error)) {
+          return { success: false, error };
+        }
       }
-      return { success: false, error };
+      return { success: true };
     } catch (error: unknown) {
       return {
         success: false,
@@ -349,28 +385,27 @@ export class LLMValidationService {
   }
 
   private static async validatePerplexity(apiKey: string): Promise<{ success: boolean; error?: string }> {
-    // Perplexity has no model list, so this sends a test message. A retired
-    // test model does not fail the key.
+    // Perplexity has no model list, so this sends a test message to its
+    // cheapest registry models.
     return this.validateWithTestMessage('perplexity', 'Perplexity', 'https://api.perplexity.ai/chat/completions', {
       'Authorization': `Bearer ${apiKey}`,
       'X-Pplx-Integration': 'nexus'
-    }, {
-      model: 'sonar',
+    }, this.cheapestTestModels(PERPLEXITY_MODELS), model => ({
+      model,
       messages: [{ role: 'user', content: 'Hi' }],
       max_tokens: 5
-    });
+    }));
   }
 
   private static async validateRequesty(apiKey: string): Promise<{ success: boolean; error?: string }> {
     // Requesty answers a bad key on its model list with the same 403 as other
-    // refusals, so this sends a test message. A retired test model does not
-    // fail the key.
+    // refusals, so this sends a test message to its cheapest registry models.
     return this.validateWithTestMessage('requesty', 'Requesty', 'https://router.requesty.ai/v1/chat/completions', {
       'Authorization': `Bearer ${apiKey}`
-    }, {
-      model: 'openai/gpt-5-nano',
+    }, this.cheapestTestModels(REQUESTY_MODELS), model => ({
+      model,
       messages: [{ role: 'user', content: 'Hi' }],
       max_completion_tokens: 5
-    });
+    }));
   }
 }

tests/unit/LLMValidationService.test.ts

@@ -7,7 +7,12 @@
  */
 
 import { __setRequestUrlMock } from 'obsidian';
+import { readFileSync } from 'fs';
+import { join } from 'path';
 import { LLMValidationService } from '../../src/services/llm/validation/ValidationService';
+import type { ModelSpec } from '../../src/services/llm/adapters/modelTypes';
+import { PERPLEXITY_MODELS } from '../../src/services/llm/adapters/perplexity/PerplexityModels';
+import { REQUESTY_MODELS } from '../../src/services/llm/adapters/requesty/RequestyModels';
 
 const mockRequestUrl = jest.fn();
 
@@ -84,19 +89,44 @@ describe('LLMValidationService', () => {
     await expect(validator('validateOpenAI')('restricted-key')).resolves.toEqual({ success: true });
   });
 
-  describe.each([['validatePerplexity'], ['validateRequesty']])('%s', (name) => {
-    it('accepts the key when only the test model is unavailable', async () => {
-      mockRequestUrl.mockResolvedValue({
-        status: 404,
-        json: { error: { message: 'No endpoints found for openai/gpt-5-nano.' } },
-        text: '',
-        headers: {}
-      });
+  describe.each([
+    ['validatePerplexity', PERPLEXITY_MODELS],
+    ['validateRequesty', REQUESTY_MODELS]
+  ] as const)('%s', (name, registry) => {
+    const expected = LLMValidationService.cheapestTestModels(registry);
+    const sentModels = () => mockRequestUrl.mock.calls.map(call => JSON.parse(call[0].body).model);
+    const modelGone = (model: string) => ({
+      status: 404,
+      json: { error: { message: `No endpoints found for ${model}.` } },
+      text: '',
+      headers: {}
+    });
+
+    it('tests with the cheapest model in its own registry', async () => {
+      mockRequestUrl.mockResolvedValue({ status: 200, json: {}, text: '{}', headers: {} });
+
+      await expect(validator(name)('test-key')).resolves.toEqual({ success: true });
+      expect(sentModels()).toEqual([expected[0]]);
+      expect(registry.map(model => model.apiName)).toContain(expected[0]);
+    });
+
+    it('moves on to the next cheapest model when one is gone', async () => {
+      mockRequestUrl
+        .mockResolvedValueOnce(modelGone(expected[0]))
+        .mockResolvedValueOnce({ status: 200, json: {}, text: '{}', headers: {} });
+
+      await expect(validator(name)('test-key')).resolves.toEqual({ success: true });
+      expect(sentModels()).toEqual(expected.slice(0, 2));
+    });
+
+    it('accepts the key when every test model is gone, because only the models were refused', async () => {
+      mockRequestUrl.mockImplementation(async (request: { body: string }) => modelGone(JSON.parse(request.body).model));
 
       await expect(validator(name)('test-key')).resolves.toEqual({ success: true });
+      expect(sentModels()).toEqual(expected);
     });
 
-    it('rejects a key the provider refuses', async () => {
+    it('stops at the first refusal of the key', async () => {
       mockRequestUrl.mockResolvedValue({
         status: 401,
         json: { error: { message: 'Invalid model access token' } },
@@ -105,6 +135,7 @@ describe('LLMValidationService', () => {
       });
 
       await expect(validator(name)('bad-key')).resolves.toEqual({ success: false, error: 'Invalid model access token' });
+      expect(mockRequestUrl).toHaveBeenCalledTimes(1);
     });
 
     it('rejects a 400 that blames the key even when it mentions a model', async () => {
@@ -118,4 +149,27 @@ describe('LLMValidationService', () => {
       await expect(validator(name)('bad-key')).resolves.toEqual({ success: false, error: 'Invalid API key for model sonar' });
     });
   });
+
+  describe('cheapestTestModels', () => {
+    const spec = (apiName: string, input: number, output: number) =>
+      ({ apiName, inputCostPerMillion: input, outputCostPerMillion: output }) as ModelSpec;
+
+    it('orders by price, puts unpriced models last, drops duplicates and caps the attempts', () => {
+      expect(LLMValidationService.cheapestTestModels([
+        spec('free', 0, 0),
+        spec('pricey', 5, 25),
+        spec('cheap', 0.1, 0.4),
+        spec('cheap', 0.1, 0.4),
+        spec('middle', 1, 4)
+      ])).toEqual(['cheap', 'middle', 'pricey']);
+    });
+  });
+
+  it('never names a model in a key check: test models come from the registry', () => {
+    // A hardcoded test model is how the OpenRouter check broke when its free
+    // model was removed. Checks must be model-free or use cheapestTestModels.
+    const source = readFileSync(join(__dirname, '../../src/services/llm/validation/ValidationService.ts'), 'utf8');
+    expect(source).not.toMatch(/model\s*:\s*['"`]/);
+    expect(source).not.toMatch(/\/models\/[a-z0-9.-]+:generateContent/i);
+  });
 });