chore: add more accuracy models via Grove MCP-491

.github/workflows/accuracy-tests.yml

@@ -18,8 +18,7 @@ jobs:
       pull-requests: write
     if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'accuracy-tests')
     env:
-      MDB_OPEN_AI_API_KEY: ${{ secrets.ACCURACY_OPEN_AI_API_KEY }}
-      MDB_GEMINI_API_KEY: ${{ secrets.ACCURACY_GEMINI_API_KEY }}
+      MDB_GROVE_API_KEY: ${{ secrets.ACCURACY_GROVE_API_KEY }}
       MDB_AZURE_OPEN_AI_API_KEY: ${{ secrets.ACCURACY_AZURE_OPEN_AI_API_KEY }}
       MDB_AZURE_OPEN_AI_API_URL: ${{ vars.ACCURACY_AZURE_OPEN_AI_API_URL_V2 }}
       MDB_ACCURACY_MDB_URL: ${{ secrets.ACCURACY_MDB_CONNECTION_STRING }}

.vscode/launch.json

@@ -44,6 +44,29 @@
         "scripts/generate/index.ts"
       ],
       "cwd": "${workspaceFolder}"
+    },
+    {
+      "type": "node",
+      "request": "launch",
+      "name": "Debug Accuracy Tests",
+      "autoAttachChildProcesses": true,
+      "skipFiles": ["<node_internals>/**"],
+      "smartStep": true,
+      "runtimeExecutable": "pnpm",
+      "runtimeArgs": ["exec", "vitest"],
+      "args": [
+        "--config", "vitest.config.ts",
+        "--project", "accuracy",
+        "--coverage=false",
+        "--max-workers=1",
+        "--run"
+      ],
+      "cwd": "${workspaceFolder}",
+      "env": {
+        "MDB_ACCURACY_RUN_ID": "debug-run",
+        "MDB_MCP_API_CLIENT_ID": "test-atlas-client-id",
+        "MDB_MCP_API_CLIENT_SECRET": "test-atlas-client-secret"
+      }
     }
   ]
 }

package.json

@@ -101,6 +101,7 @@
   },
   "license": "Apache-2.0",
   "devDependencies": {
+    "@ai-sdk/anthropic": "^3.0.57",
     "@ai-sdk/azure": "^3.0.54",
     "@ai-sdk/google": "^3.0.64",
     "@ai-sdk/mcp": "^1.0.36",

scripts/accuracy/runAccuracyTests.sh

@@ -3,10 +3,11 @@
 export MDB_ACCURACY_RUN_ID=$(pnpm dlx uuid v4)
 
 # For providing access tokens for different LLM providers
-# export MDB_OPEN_AI_API_KEY=""
-# export MDB_GEMINI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_KEY=""
 # export MDB_AZURE_OPEN_AI_API_URL=""
+# export MDB_GROVE_API_KEY=""
+# Optional comma-separated model allow list, for example:
+# export MDB_ACCURACY_MODEL_ALLOWLIST="gpt-5.5,claude-sonnet-4-6,Kimi-K2.6,grok-4-20-reasoning,deepseek-r1-0528"
 
 # For providing Atlas API credentials (required for Atlas tools)
 # Set dummy values for testing (allows Atlas tools to be registered for mocking)
@@ -22,7 +23,7 @@ export MDB_MCP_API_CLIENT_SECRET=${MDB_MCP_API_CLIENT_SECRET:-"test-atlas-client
 # specified in the command line. Such as:
 # pnpm run test:accuracy -- tests/accuracy/some-test.test.ts
 echo "Running accuracy tests with MDB_ACCURACY_RUN_ID '$MDB_ACCURACY_RUN_ID'"
-vitest --config vitest.config.ts --project=accuracy --coverage=false --max-workers=2 --run "$@"
+vitest --config vitest.config.ts --project=accuracy --coverage=false --max-workers=8 --run "$@"
 
 # Preserving the exit code from test run to correctly notify in the CI
 # environments when the tests fail.

tests/accuracy/createDeployment.test.ts

@@ -45,7 +45,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "local-cluster",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -63,7 +63,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "local-cluster",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -80,7 +80,7 @@ describeAccuracyTests([
                 toolName: "atlas-local-create-deployment",
                 parameters: {
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -102,7 +102,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "new-database",
                     loadSampleData: Matcher.anyOf(Matcher.undefined, Matcher.boolean()),
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -142,7 +142,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "sample-cluster",
                     loadSampleData: true,
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],
@@ -160,7 +160,7 @@ describeAccuracyTests([
                 parameters: {
                     deploymentName: "empty-cluster",
                     loadSampleData: false,
-                    imageTag: "preview",
+                    imageTag: Matcher.anyOf(Matcher.undefined, Matcher.string()),
                 },
             },
         ],

tests/accuracy/sdk/accuracyResultStorage/diskStorage.ts

@@ -28,6 +28,12 @@ export class DiskBasedResultStorage implements AccuracyResultStorage {
 
     async updateRunStatus(commitSHA: string, runId: string, status: AccuracyRunStatuses): Promise<void> {
         const resultFilePath = this.getAccuracyResultFilePath(commitSHA, runId);
+        try {
+            await fs.access(resultFilePath);
+        } catch {
+            // No results file means no tests completed - nothing to update.
+            return;
+        }
         await this.withFileLock(resultFilePath, async () => {
             const accuracyResult = await this.getAccuracyResultWithoutLock(resultFilePath);
             if (!accuracyResult) {

tests/accuracy/sdk/accuracyTestingClient.ts

@@ -94,13 +94,17 @@ export class AccuracyTestingClient {
 
         const args = [MCP_SERVER_CLI_SCRIPT, mdbConnectionString, ...additionalArgs];
 
+        // Strip MDB_MCP_* env vars so the local MCP config (e.g.
+        // readOnly, disabledTools) doesn't bleed into the test server process.
+        const env = Object.fromEntries(
+            Object.entries(process.env).filter(([key]) => !key.startsWith("MDB_MCP_"))
+        ) as Record<string, string>;
+        env.DO_NOT_TRACK = "1";
+
         const clientTransport = new StdioClientTransport({
             command: process.execPath,
             args,
-            env: {
-                ...process.env,
-                DO_NOT_TRACK: "1",
-            },
+            env,
         });
 
         const client = await createMCPClient({

tests/accuracy/sdk/models.ts

@@ -1,7 +1,7 @@
 import type { LanguageModel } from "ai";
-import { createGoogleGenerativeAI } from "@ai-sdk/google";
 import { createAzure } from "@ai-sdk/azure";
 import { createOpenAI } from "@ai-sdk/openai";
+import { createAnthropic } from "@ai-sdk/anthropic";
 
 export interface Model<VercelModel extends LanguageModel = LanguageModel> {
     readonly modelName: string;
@@ -11,68 +11,112 @@ export interface Model<VercelModel extends LanguageModel = LanguageModel> {
     getModel(): VercelModel;
 }
 
-export class OpenAIModel implements Model {
-    readonly provider = "OpenAI";
+class GroveOpenAICompatibleModel implements Model {
+    readonly provider: string;
     readonly displayName: string;
 
-    constructor(readonly modelName: string) {
+    constructor(
+        readonly modelName: string,
+        providerName: string
+    ) {
+        this.provider = `Grove/${providerName}`;
         this.displayName = `${this.provider} - ${modelName}`;
     }
 
     isAvailable(): boolean {
-        return !!process.env.MDB_OPEN_AI_API_KEY;
+        return !!process.env.MDB_GROVE_API_KEY;
     }
 
     getModel(): LanguageModel {
         return createOpenAI({
-            apiKey: process.env.MDB_OPEN_AI_API_KEY,
+            baseURL: "https://grove-gateway-prod.azure-api.net/grove-foundry-prod/openai/v1",
+            apiKey: process.env.MDB_GROVE_API_KEY,
+            headers: {
+                "api-key": process.env.MDB_GROVE_API_KEY ?? "",
+            },
         }).chat(this.modelName);
     }
 }
 
-export class AzureOpenAIModel implements Model {
-    readonly provider = "Azure";
+class GroveAnthropicModel implements Model {
+    readonly provider = "Grove/Anthropic";
     readonly displayName: string;
 
     constructor(readonly modelName: string) {
         this.displayName = `${this.provider} - ${modelName}`;
     }
 
     isAvailable(): boolean {
-        return !!process.env.MDB_AZURE_OPEN_AI_API_KEY && !!process.env.MDB_AZURE_OPEN_AI_API_URL;
+        return !!process.env.MDB_GROVE_API_KEY;
     }
 
     getModel(): LanguageModel {
-        return createAzure({
-            baseURL: process.env.MDB_AZURE_OPEN_AI_API_URL,
-            apiKey: process.env.MDB_AZURE_OPEN_AI_API_KEY,
-            useDeploymentBasedUrls: true,
-            apiVersion: "2024-12-01-preview",
+        return createAnthropic({
+            baseURL: "https://grove-gateway-prod.azure-api.net/grove-foundry-prod/anthropic/v1",
+            apiKey: process.env.MDB_GROVE_API_KEY,
+            headers: {
+                "api-key": process.env.MDB_GROVE_API_KEY ?? "",
+            },
         }).chat(this.modelName);
     }
 }
 
-export class GeminiModel implements Model {
-    readonly provider = "Google";
+class AzureOpenAIModel implements Model {
+    readonly provider = "Azure";
     readonly displayName: string;
 
     constructor(readonly modelName: string) {
         this.displayName = `${this.provider} - ${modelName}`;
     }
 
     isAvailable(): boolean {
-        return !!process.env.MDB_GEMINI_API_KEY;
+        return !!process.env.MDB_AZURE_OPEN_AI_API_KEY && !!process.env.MDB_AZURE_OPEN_AI_API_URL;
     }
 
     getModel(): LanguageModel {
-        return createGoogleGenerativeAI({
-            apiKey: process.env.MDB_GEMINI_API_KEY,
+        return createAzure({
+            baseURL: process.env.MDB_AZURE_OPEN_AI_API_URL,
+            apiKey: process.env.MDB_AZURE_OPEN_AI_API_KEY,
+            useDeploymentBasedUrls: true,
+            apiVersion: "2024-12-01-preview",
         }).chat(this.modelName);
     }
 }
 
-const ALL_TESTABLE_MODELS: Model[] = [new AzureOpenAIModel("gpt-4o")];
+const ALL_TESTABLE_MODELS: Model[] = [
+    new GroveOpenAICompatibleModel("gpt-5.5", "OpenAI"),
+    new GroveOpenAICompatibleModel("Kimi-K2.6", "Kimi"),
+    new GroveOpenAICompatibleModel("grok-4-20-reasoning", "Grok"),
+    new GroveOpenAICompatibleModel("deepseek-r1-0528", "DeepSeek"),
+    new GroveAnthropicModel("claude-sonnet-4-6"),
+    new AzureOpenAIModel("gpt-4o"),
+];
+
+function getConfiguredModelAllowList(): Set<string> | null {
+    const modelAllowList = process.env.MDB_ACCURACY_MODEL_ALLOWLIST?.trim();
+    if (!modelAllowList) {
+        return null;
+    }
+
+    const models = modelAllowList.split(",").map((item) => item.trim());
+    if (!models.length) {
+        return null;
+    }
+
+    return new Set(models);
+}
 
 export function getAvailableModels(): Model[] {
-    return ALL_TESTABLE_MODELS.filter((model) => model.isAvailable());
+    const allowList = getConfiguredModelAllowList();
+    return ALL_TESTABLE_MODELS.filter((model) => {
+        if (!model.isAvailable()) {
+            return false;
+        }
+
+        if (!allowList) {
+            return true;
+        }
+
+        return allowList.has(model.modelName);
+    });
 }