From 7ed01ed9c1a18c5e37569a55d956488fbd1d9b2e Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Sat, 28 Jun 2025 16:14:03 +0200 Subject: [PATCH 01/16] chore: LangChain based accuracy tests --- package-lock.json | 510 +++++++++++++++++- package.json | 6 + tests/accuracy/list-databases.test.ts | 26 + tests/accuracy/sdk/accuracy-scorers.ts | 125 +++++ tests/accuracy/sdk/describe-accuracy-tests.ts | 51 ++ tests/accuracy/sdk/models.ts | 62 +++ tests/accuracy/sdk/test-tools.ts | 153 ++++++ tests/accuracy/sdk/tool-calling-agent.ts | 36 ++ 8 files changed, 965 insertions(+), 4 deletions(-) create mode 100644 tests/accuracy/list-databases.test.ts create mode 100644 tests/accuracy/sdk/accuracy-scorers.ts create mode 100644 tests/accuracy/sdk/describe-accuracy-tests.ts create mode 100644 tests/accuracy/sdk/models.ts create mode 100644 tests/accuracy/sdk/test-tools.ts create mode 100644 tests/accuracy/sdk/tool-calling-agent.ts diff --git a/package-lock.json b/package-lock.json index 7c3ddb32..47508a4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,10 @@ "devDependencies": { "@eslint/js": "^9.24.0", "@jest/globals": "^30.0.0", + "@langchain/core": "^0.3.61", + "@langchain/google-genai": "^0.2.14", + "@langchain/ollama": "^0.2.3", + "@langchain/openai": "^0.5.16", "@modelcontextprotocol/inspector": "^0.14.0", "@redocly/cli": "^1.34.2", "@types/jest": "^29.5.14", @@ -46,6 +50,7 @@ "jest": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-extended": "^6.0.0", + "langchain": "^0.3.29", "mongodb-runner": "^5.8.2", "openapi-types": "^12.1.3", "openapi-typescript": "^7.6.1", @@ -54,6 +59,7 @@ "tsx": "^4.19.3", "typescript": "^5.8.2", "typescript-eslint": "^8.29.1", + "uuid": "^11.1.0", "yaml": "^2.7.1" }, "engines": { @@ -1267,6 +1273,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "dev": true, + "license": "MIT" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -2039,6 +2052,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@google/generative-ai": { + "version": "0.24.1", + "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", + "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@hapi/boom": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-10.0.1.tgz", @@ -3199,6 +3222,152 @@ "jsep": "^0.4.0||^1.0.0" } }, + "node_modules/@langchain/core": { + "version": "0.3.61", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.61.tgz", + "integrity": "sha512-4O7fw5SXNSE+uBnathLQrhm3t+7dZGagt/5kt37A+pXw0AkudxEBvveg73sSnpBd9SIz3/Vc7F4k8rCKXGbEDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.3.33", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.25.32", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/core/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@langchain/google-genai": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.2.14.tgz", + "integrity": "sha512-gKe/T2LNh8wSSMJOaFmYd8cwQnDSXKtVtC6a7CFoq5nWuh0bKzhItM/7bue1aMN8mlKfB2G1HCwxhaZoSpS/DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@google/generative-ai": "^0.24.0", + "uuid": "^11.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.58 <0.4.0" + } + }, + "node_modules/@langchain/ollama": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.2.3.tgz", + "integrity": "sha512-1Obe45jgQspqLMBVlayQbGdywFmri8DgmGRdzNu0li56cG5RReYlRCFVDZBRMMvF9JhsP5eXRyfyivtKfITHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ollama": "^0.5.12", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.58 <0.4.0" + } + }, + "node_modules/@langchain/ollama/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@langchain/openai": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.5.16.tgz", + "integrity": "sha512-TqzPE3PM0bMkQi53qs8vCFkwaEp3VgwGw+s1e8Nas5ICCZZtc2XqcDPz4hf2gpo1k7/AZd6HuPlAsDy6wye9Qw==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tiktoken": "^1.0.12", + "openai": "^5.3.0", + "zod": "^3.25.32" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.3.58 <0.4.0" + } + }, + "node_modules/@langchain/textsplitters": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.1.0.tgz", + "integrity": "sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "js-tiktoken": "^1.0.12" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": ">=0.2.21 <0.4.0" + } + }, "node_modules/@modelcontextprotocol/inspector": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/inspector/-/inspector-0.14.1.tgz", @@ -5729,6 +5898,19 @@ "node": ">=18.0.0" } }, + "node_modules/@smithy/middleware-retry/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@smithy/middleware-serde": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-4.0.3.tgz", @@ -6272,6 +6454,13 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/simple-oauth2": { "version": "5.0.7", "resolved": "https://registry.npmjs.org/@types/simple-oauth2/-/simple-oauth2-5.0.7.tgz", @@ -6301,6 +6490,13 @@ "license": "MIT", "optional": true }, + "node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -7747,6 +7943,16 @@ "node": ">=12" } }, + "node_modules/console-table-printer": { + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.14.6.tgz", + "integrity": "sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "simple-wcswidth": "^1.0.1" + } + }, "node_modules/content-disposition": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", @@ -7937,6 +8143,16 @@ } } }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/decko": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decko/-/decko-1.2.0.tgz", @@ -11084,6 +11300,16 @@ "node": ">=0.10.0" } }, + "node_modules/js-tiktoken": { + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.20.tgz", + "integrity": "sha512-Xlaqhhs8VfCd6Sh7a1cFkZHQbYTLCwVJJWiHVxBYzLPxW0XsoxBy1hitmjkdIjD3Aon5BXLHFwU5O8WUx6HH+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -11248,6 +11474,154 @@ "node": ">=6" } }, + "node_modules/langchain": { + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.3.29.tgz", + "integrity": "sha512-L389pKlApVJPqu4hp58qY6NZAobI+MFPoBjSfjT1z3mcxtB68wLFGhaH4DVsTVg21NYO+0wTEoz24BWrxu9YGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/openai": ">=0.1.0 <0.6.0", + "@langchain/textsplitters": ">=0.0.0 <0.2.0", + "js-tiktoken": "^1.0.12", + "js-yaml": "^4.1.0", + "jsonpointer": "^5.0.1", + "langsmith": "^0.3.33", + "openapi-types": "^12.1.3", + "p-retry": "4", + "uuid": "^10.0.0", + "yaml": "^2.2.1", + "zod": "^3.25.32" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/anthropic": "*", + "@langchain/aws": "*", + "@langchain/cerebras": "*", + "@langchain/cohere": "*", + "@langchain/core": ">=0.3.58 <0.4.0", + "@langchain/deepseek": "*", + "@langchain/google-genai": "*", + "@langchain/google-vertexai": "*", + "@langchain/google-vertexai-web": "*", + "@langchain/groq": "*", + "@langchain/mistralai": "*", + "@langchain/ollama": "*", + "@langchain/xai": "*", + "axios": "*", + "cheerio": "*", + "handlebars": "^4.7.8", + "peggy": "^3.0.2", + "typeorm": "*" + }, + "peerDependenciesMeta": { + "@langchain/anthropic": { + "optional": true + }, + "@langchain/aws": { + "optional": true + }, + "@langchain/cerebras": { + "optional": true + }, + "@langchain/cohere": { + "optional": true + }, + "@langchain/deepseek": { + "optional": true + }, + "@langchain/google-genai": { + "optional": true + }, + "@langchain/google-vertexai": { + "optional": true + }, + "@langchain/google-vertexai-web": { + "optional": true + }, + "@langchain/groq": { + "optional": true + }, + "@langchain/mistralai": { + "optional": true + }, + "@langchain/ollama": { + "optional": true + }, + "@langchain/xai": { + "optional": true + }, + "axios": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "handlebars": { + "optional": true + }, + "peggy": { + "optional": true + }, + "typeorm": { + "optional": true + } + } + }, + "node_modules/langchain/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/langsmith": { + "version": "0.3.34", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.34.tgz", + "integrity": "sha512-rxYuuypqaSzIuNjZMTsCVAgG0cYdI516dwuKn58bu4YuBRlLaLeNlHewRyoqP9lrEAlpkekCV9fUwwZ7lO8f2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/uuid": "^10.0.0", + "chalk": "^4.1.2", + "console-table-printer": "^2.12.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "openai": "*" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + } + } + }, + "node_modules/langsmith/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -12018,6 +12392,16 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nan": { "version": "2.22.2", "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", @@ -12360,6 +12744,16 @@ "node": "^10.13.0 || >=12.0.0" } }, + "node_modules/ollama": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.16.tgz", + "integrity": "sha512-OEbxxOIUZtdZgOaTPAULo051F5y+Z1vosxEYOoABPnQKeW7i4O8tJNlxCB+xioyoorVqgjkdj+TA1f1Hy2ug/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-fetch": "^3.6.20" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -12414,6 +12808,28 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "5.8.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.8.2.tgz", + "integrity": "sha512-8C+nzoHYgyYOXhHGN6r0fcb4SznuEn1R7YZMvlqDbnCuE0FM2mm3T1HiYW6WIcMS/F1Of2up/cSPjLPaWt0X9Q==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, "node_modules/openapi-fetch": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.14.0.tgz", @@ -12615,6 +13031,16 @@ "dev": true, "license": "MIT" }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -12647,6 +13073,57 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue/node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -13652,6 +14129,16 @@ "node": ">=10" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -14186,6 +14673,13 @@ "joi": "^17.6.4" } }, + "node_modules/simple-wcswidth": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", + "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", + "dev": true, + "license": "MIT" + }, "node_modules/simple-websocket": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/simple-websocket/-/simple-websocket-9.1.0.tgz", @@ -15433,16 +15927,17 @@ } }, "node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.0.tgz", + "integrity": "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==", + "dev": true, "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" ], "license": "MIT", "bin": { - "uuid": "dist/bin/uuid" + "uuid": "dist/esm/bin/uuid" } }, "node_modules/v8-compile-cache-lib": { @@ -15504,6 +15999,13 @@ "node": ">=12" } }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", + "dev": true, + "license": "MIT" + }, "node_modules/whatwg-url": { "version": "14.2.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", diff --git a/package.json b/package.json index ca262abc..9fb6a9b5 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,10 @@ "devDependencies": { "@eslint/js": "^9.24.0", "@jest/globals": "^30.0.0", + "@langchain/core": "^0.3.61", + "@langchain/google-genai": "^0.2.14", + "@langchain/ollama": "^0.2.3", + "@langchain/openai": "^0.5.16", "@modelcontextprotocol/inspector": "^0.14.0", "@redocly/cli": "^1.34.2", "@types/jest": "^29.5.14", @@ -49,6 +53,7 @@ "jest": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-extended": "^6.0.0", + "langchain": "^0.3.29", "mongodb-runner": "^5.8.2", "openapi-types": "^12.1.3", "openapi-typescript": "^7.6.1", @@ -57,6 +62,7 @@ "tsx": "^4.19.3", "typescript": "^5.8.2", "typescript-eslint": "^8.29.1", + "uuid": "^11.1.0", "yaml": "^2.7.1" }, "dependencies": { diff --git a/tests/accuracy/list-databases.test.ts b/tests/accuracy/list-databases.test.ts new file mode 100644 index 00000000..ae3f6c7d --- /dev/null +++ b/tests/accuracy/list-databases.test.ts @@ -0,0 +1,26 @@ +import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; +import { getAvailableModels } from "./sdk/models.js"; + +describeAccuracyTests("list-databases", getAvailableModels(), [ + { + prompt: "Assume that you're already connected. How many collections are there in sample_mflix database", + mockedTools: { + "list-collections": function listCollections() { + return { + content: [ + { + type: "text", + text: "Name: coll1", + }, + ], + }; + }, + }, + expectedToolCalls: [ + { + toolName: "list-collections", + parameters: { database: "sample_mflix" }, + }, + ], + }, +]); diff --git a/tests/accuracy/sdk/accuracy-scorers.ts b/tests/accuracy/sdk/accuracy-scorers.ts new file mode 100644 index 00000000..bf92eead --- /dev/null +++ b/tests/accuracy/sdk/accuracy-scorers.ts @@ -0,0 +1,125 @@ +export type ToolCall = { + toolCallId: string; + toolName: string; + parameters: unknown; +}; +export type ExpectedToolCall = Omit; + +export function toolCallingAccuracyScorer(expectedToolCalls: ExpectedToolCall[], actualToolCalls: ToolCall[]): number { + if (actualToolCalls.length < expectedToolCalls.length) { + return 0; + } + + const possibleScore = actualToolCalls.length > expectedToolCalls.length ? 0.75 : 1; + const checkedToolCallIds = new Set(); + for (const expectedToolCall of expectedToolCalls) { + const matchingActualToolCall = actualToolCalls.find( + (actualToolCall) => + actualToolCall.toolName === expectedToolCall.toolName && + !checkedToolCallIds.has(actualToolCall.toolCallId) + ); + + if (!matchingActualToolCall) { + return 0; + } + + checkedToolCallIds.add(matchingActualToolCall.toolCallId); + } + + return possibleScore; +} + +export function parameterMatchingAccuracyScorer( + expectedToolCalls: ExpectedToolCall[], + actualToolCalls: ToolCall[] +): number { + if (expectedToolCalls.length === 0) { + return 1; + } + + const toolCallScores: number[] = []; + const checkedToolCallIds = new Set(); + + for (const expectedToolCall of expectedToolCalls) { + const matchingActualToolCall = actualToolCalls.find( + (actualToolCall) => + actualToolCall.toolName === expectedToolCall.toolName && + !checkedToolCallIds.has(actualToolCall.toolCallId) + ); + + if (!matchingActualToolCall) { + toolCallScores.push(0); + continue; + } + + checkedToolCallIds.add(matchingActualToolCall.toolCallId); + const score = compareParams(expectedToolCall.parameters, matchingActualToolCall.parameters); + toolCallScores.push(score); + } + + const totalScore = toolCallScores.reduce((sum, score) => sum + score, 0); + return totalScore / toolCallScores.length; +} + +/** + * Recursively compares expected and actual parameters and returns a score. + * - 1: Perfect match. + * - 0.75: All expected parameters are present and match, but there are extra actual parameters. + * - 0: Missing parameters or mismatched values. + */ +function compareParams(expected: unknown, actual: unknown): number { + if (expected === null || expected === undefined) { + return actual === null || actual === undefined ? 1 : 0; + } + if (actual === null || actual === undefined) { + return 0; + } + + if (Array.isArray(expected)) { + if (!Array.isArray(actual) || actual.length < expected.length) { + return 0; + } + let minScore = 1; + for (let i = 0; i < expected.length; i++) { + minScore = Math.min(minScore, compareParams(expected[i], actual[i])); + } + if (minScore === 0) { + return 0; + } + if (actual.length > expected.length) { + minScore = Math.min(minScore, 0.75); + } + return minScore; + } + + if (typeof expected === "object") { + if (typeof actual !== "object" || Array.isArray(actual)) { + return 0; + } + const expectedKeys = Object.keys(expected as Record); + const actualKeys = Object.keys(actual as Record); + + let minScore = 1; + for (const key of expectedKeys) { + if (!Object.prototype.hasOwnProperty.call(actual, key)) { + return 0; + } + minScore = Math.min( + minScore, + compareParams((expected as Record)[key], (actual as Record)[key]) + ); + } + + if (minScore === 0) { + return 0; + } + + if (actualKeys.length > expectedKeys.length) { + minScore = Math.min(minScore, 0.75); + } + return minScore; + } + + // eslint-disable-next-line eqeqeq + return expected == actual ? 1 : 0; +} diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts new file mode 100644 index 00000000..0ec4bb64 --- /dev/null +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -0,0 +1,51 @@ +import { AgentExecutor } from "langchain/agents"; +import { Tool } from "@modelcontextprotocol/sdk/types.js"; +import { discoverMongoDBTools, TestTools, ToolResultGenerators } from "./test-tools.js"; +import { TestableModels } from "./models.js"; +import { getToolCallingAgent } from "./tool-calling-agent.js"; +import { ExpectedToolCall, parameterMatchingAccuracyScorer, toolCallingAccuracyScorer } from "./accuracy-scorers.js"; + +interface AccuracyTestConfig { + prompt: string; + expectedToolCalls: ExpectedToolCall[]; + mockedTools: ToolResultGenerators; +} + +export function describeAccuracyTests( + suiteName: string, + models: TestableModels, + accuracyTestConfigs: AccuracyTestConfig[] +) { + const eachModel = describe.each(models); + const eachTest = it.each(accuracyTestConfigs); + + eachModel(`$modelName - ${suiteName}`, function (model) { + let mcpTools: Tool[]; + let testTools: TestTools; + let agent: AgentExecutor; + + beforeAll(async () => { + mcpTools = await discoverMongoDBTools(); + }); + + beforeEach(() => { + testTools = new TestTools(mcpTools); + const transformToolResult = model.transformToolResult.bind(model); + agent = getToolCallingAgent(model, testTools.langChainTools(transformToolResult)); + }); + + eachTest("$prompt", async function (testConfig) { + testTools.mockTools(testConfig.mockedTools); + const conversation = await agent.invoke({ input: testConfig.prompt }); + console.log("conversation", conversation); + const toolCalls = testTools.getToolCalls(); + console.log("?????? toolCalls", toolCalls); + console.log("???? expected", testConfig.expectedToolCalls); + const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); + const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); + + expect(toolCallingAccuracy).not.toEqual(0); + expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); + }); + }); +} diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts new file mode 100644 index 00000000..d370633f --- /dev/null +++ b/tests/accuracy/sdk/models.ts @@ -0,0 +1,62 @@ +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; +import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; +import { ChatOllama } from "@langchain/ollama"; +import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; + +type ToolResultForOllama = string; +export type AcceptableToolResponse = CallToolResult | ToolResultForOllama; + +export interface Model { + isAvailable(): boolean; + getLangChainModel(): M; + transformToolResult(callToolResult: CallToolResult): T; +} + +export class GeminiModel implements Model { + constructor(readonly modelName: string) {} + + isAvailable(): boolean { + return !!process.env.MDB_GEMINI_API_KEY; + } + + getLangChainModel(): ChatGoogleGenerativeAI { + return new ChatGoogleGenerativeAI({ + model: this.modelName, + apiKey: process.env.MDB_GEMINI_API_KEY, + }); + } + + transformToolResult(callToolResult: CallToolResult) { + return callToolResult; + } +} + +export class OllamaModel implements Model { + constructor(readonly modelName: string) {} + + isAvailable(): boolean { + return !!process.env.MDB_GEMINI_API_KEY; + } + + getLangChainModel(): ChatOllama { + return new ChatOllama({ + model: this.modelName, + }); + } + + transformToolResult(callToolResult: CallToolResult): ToolResultForOllama { + return JSON.stringify(callToolResult); + } +} + +const ALL_TESTABLE_MODELS = [ + // new GeminiModel("gemini-1.5-flash"), + // new GeminiModel("gemini-2.0-flash"), + new OllamaModel("qwen3:latest"), +]; + +export type TestableModels = ReturnType; + +export function getAvailableModels() { + return ALL_TESTABLE_MODELS.filter((model) => model.isAvailable()); +} diff --git a/tests/accuracy/sdk/test-tools.ts b/tests/accuracy/sdk/test-tools.ts new file mode 100644 index 00000000..82719454 --- /dev/null +++ b/tests/accuracy/sdk/test-tools.ts @@ -0,0 +1,153 @@ +import { jest } from "@jest/globals"; +import { v4 as uuid } from "uuid"; +import { DynamicTool, tool as langChainTool } from "@langchain/core/tools"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { CallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js"; + +import { InMemoryTransport } from "../../integration/inMemoryTransport.js"; +import { defaultTestConfig } from "../../integration/helpers.js"; +import { Session } from "../../../src/session.js"; +import { Telemetry } from "../../../src/telemetry/telemetry.js"; +import { Server } from "../../../src/server.js"; +import { AcceptableToolResponse } from "./models.js"; +import { ToolCall } from "./accuracy-scorers.js"; + +type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult; +type MockedToolResultGeneratorFn = jest.MockedFunction; +type MockedTools = Record; +export type ToolResultGenerators = Record; +export type LangChainTool = DynamicTool; +export type ToolResultTransformer = (toolResult: CallToolResult) => T; + +export class TestTools { + private mockedTools: MockedTools = {}; + private recordedToolCalls: ToolCall[] = []; + + constructor(private readonly mcpTools: Tool[]) { + for (const mcpTool of mcpTools) { + this.mockedTools[mcpTool.name] = jest.fn().mockReturnValue({ + content: [ + { + type: "text", + text: `Mock implementation for tool - ${mcpTool.name} not present`, + }, + ], + isError: true, + }); + } + } + + getToolCalls() { + return this.recordedToolCalls; + } + + mockTools(toolResultGenerators: ToolResultGenerators) { + for (const toolName in toolResultGenerators) { + const toolResultGeneratorFn = toolResultGenerators[toolName]; + if (!this.mockedTools[toolName]) { + throw new Error(`Attempted to mock unrecognized tool - ${toolName}`); + } + + if (!toolResultGeneratorFn) { + // Are you happy TS? + continue; + } + this.mockedTools[toolName] = jest.fn(toolResultGeneratorFn); + } + } + + langChainTools( + transformToolResult: ToolResultTransformer + ): LangChainTool[] { + return this.mcpTools.map((mcpTool) => { + return langChainTool((...args) => { + console.log("????? args", args); + const [parameters, { runName, runId }] = args; + const toolCallId = typeof runId !== "undefined" ? `${runId}` : uuid(); + return this.langChainToolResultGenerator(`${runName}`, parameters, toolCallId, transformToolResult); + }, mcpTool); + }); + } + + private langChainToolResultGenerator( + tool: string, + parameters: unknown, + toolCallId: string, + transformToolResult: ToolResultTransformer + ): T { + this.recordedToolCalls.push({ + toolCallId: toolCallId, + toolName: tool, + parameters, + }); + const mockedToolResultGenerator = this.mockedTools[tool]; + if (!mockedToolResultGenerator) { + // log as well + return transformToolResult({ + content: [ + { + type: "text", + text: `Could not resolve tool generator for ${tool}`, + }, + ], + isError: true, + }); + } + + return transformToolResult(mockedToolResultGenerator(parameters)); + } +} + +export async function discoverMongoDBTools(): Promise { + let mcpClient: Client | undefined; + let mcpServer: Server | undefined; + try { + const serverTransport = new InMemoryTransport(); + const clientTransport = new InMemoryTransport(); + + await serverTransport.start(); + await clientTransport.start(); + + void serverTransport.output.pipeTo(clientTransport.input); + void clientTransport.output.pipeTo(serverTransport.input); + + const session = new Session({ + apiBaseUrl: defaultTestConfig.apiBaseUrl, + }); + + const telemetry = Telemetry.create(session, defaultTestConfig); + + mcpClient = new Client( + { + name: "tool-discovery-client", + version: "0.0.0", + }, + { + capabilities: {}, + } + ); + + mcpServer = new Server({ + session, + userConfig: defaultTestConfig, + telemetry, + mcpServer: new McpServer({ + name: "test-server", + version: "5.2.3", + }), + }); + + await mcpServer.connect(serverTransport); + await mcpClient.connect(clientTransport); + + return (await mcpClient.listTools()).tools; + } catch (error: unknown) { + console.error("Unexpected error occured", error); + return []; + } finally { + await mcpClient?.close(); + await mcpServer?.session?.close(); + await mcpServer?.close(); + } +} diff --git a/tests/accuracy/sdk/tool-calling-agent.ts b/tests/accuracy/sdk/tool-calling-agent.ts new file mode 100644 index 00000000..b9adedf5 --- /dev/null +++ b/tests/accuracy/sdk/tool-calling-agent.ts @@ -0,0 +1,36 @@ +import { ChatPromptTemplate } from "@langchain/core/prompts"; +import { createToolCallingAgent, AgentExecutor } from "langchain/agents"; + +import { LangChainTool } from "./test-tools.js"; +import { AcceptableToolResponse, Model } from "./models.js"; +import { BaseChatModel } from "@langchain/core/language_models/chat_models"; + +const prompt = ChatPromptTemplate.fromMessages([ + [ + "system", + [ + 'The keywords "MUST", "MUST NOT", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 (https://www.ietf.org/rfc/rfc2119.txt)', + "You are an expect AI assistant with access to a set of tools for MongoDB database operations.", + "You MUST use the most relevant tool to answer the user's request", + "When calling a tool, you MUST strictly follow its input schema and MUST provide all required arguments", + "If a task requires multiple steps, you MUST call the necessary tools in sequence", + 'If you do not know the answer or the request cannot be fulfilled, you MUST reply with "I don\'t know"', + ].join("\n"), + ], + ["human", "{input}"], + ["placeholder", "{agent_scratchpad}"], +]); + +export function getToolCallingAgent( + model: Model, + tools: LangChainTool[] +) { + const llm = model.getLangChainModel(); + const agent = createToolCallingAgent({ + llm, + tools, + prompt, + }); + const agentExecutor = new AgentExecutor({ agent, tools }); + return agentExecutor; +} From 19a881eb8a0b0a15b2ac99ea0c747c164695d5f7 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 30 Jun 2025 11:06:38 +0200 Subject: [PATCH 02/16] chore: use vercel AI SDK instead of langchain LangChain's ToolCalling agent was not providing a structured tool call response and different model providers were providing entirely different tool calls for the same tool definition which was too turbulent for us to have any accuracy baseline at all. Vercel's AI SDK pushes us forward on that problem and the tool call responses so far have always been well structured. This commit replaces LangChain based implementation with Vercel's AI SDK based implementation. --- package-lock.json | 761 +++++++----------- package.json | 9 +- tests/accuracy/sdk/agent.ts | 38 + tests/accuracy/sdk/describe-accuracy-tests.ts | 14 +- tests/accuracy/sdk/models.ts | 46 +- tests/accuracy/sdk/test-tools.ts | 106 ++- tests/accuracy/sdk/tool-calling-agent.ts | 36 - 7 files changed, 389 insertions(+), 621 deletions(-) create mode 100644 tests/accuracy/sdk/agent.ts delete mode 100644 tests/accuracy/sdk/tool-calling-agent.ts diff --git a/package-lock.json b/package-lock.json index 47508a4f..34e1c677 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,17 +31,15 @@ }, "devDependencies": { "@eslint/js": "^9.24.0", + "@himanshusinghs/google": "^1.2.11", "@jest/globals": "^30.0.0", - "@langchain/core": "^0.3.61", - "@langchain/google-genai": "^0.2.14", - "@langchain/ollama": "^0.2.3", - "@langchain/openai": "^0.5.16", "@modelcontextprotocol/inspector": "^0.14.0", "@redocly/cli": "^1.34.2", "@types/jest": "^29.5.14", "@types/node": "^22.14.0", "@types/simple-oauth2": "^5.0.7", "@types/yargs-parser": "^21.0.3", + "ai": "^4.3.16", "eslint": "^9.24.0", "eslint-config-prettier": "^10.1.2", "eslint-plugin-jest": "^29.0.1", @@ -50,8 +48,9 @@ "jest": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-extended": "^6.0.0", - "langchain": "^0.3.29", + "json-schema": "^0.4.0", "mongodb-runner": "^5.8.2", + "ollama-ai-provider": "^1.2.0", "openapi-types": "^12.1.3", "openapi-typescript": "^7.6.1", "prettier": "^3.5.3", @@ -66,6 +65,83 @@ "node": ">=20.10.0" } }, + "@himanshusinghs/ai-sdk-google": { + "extraneous": true + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -1273,13 +1349,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@cfworker/json-schema": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", - "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", - "dev": true, - "license": "MIT" - }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -2052,16 +2121,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@google/generative-ai": { - "version": "0.24.1", - "resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.24.1.tgz", - "integrity": "sha512-MqO+MLfM6kjxcKoy0p1wRzG3b4ZZXtPI+z2IE26UogS2Cm/XHO+7gGRBh6gcJsOiIVoH93UwKvW4HdgiOZCy9Q==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@hapi/boom": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@hapi/boom/-/boom-10.0.1.tgz", @@ -2109,6 +2168,54 @@ "@hapi/hoek": "^11.0.2" } }, + "node_modules/@himanshusinghs/google": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@himanshusinghs/google/-/google-1.2.11.tgz", + "integrity": "sha512-SKTFxwN9PpUHVrppFod8sF1jqys5azzsgcBVrSbc7VaazmVEnBxHQlv5/yfeZFjD3ly5Mw+AJdFfC0bxwdWBNg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.2", + "@ai-sdk/provider-utils": "2.2.6" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@himanshusinghs/google/node_modules/@ai-sdk/provider": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.2.tgz", + "integrity": "sha512-ITdgNilJZwLKR7X5TnUr1BsQW6UTX5yFp0h66Nfx8XjBYkWD9W3yugr50GOz3CnE9m/U/Cd5OyEbTMI0rgi6ZQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@himanshusinghs/google/node_modules/@ai-sdk/provider-utils": { + "version": "2.2.6", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.6.tgz", + "integrity": "sha512-sUlZ7Gnq84DCGWMQRIK8XVbkzIBnvPR1diV4v6JwPgpn5armnLI/j+rqn62MpLrU5ZCQZlDKl/Lw6ed3ulYqaA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.2", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -3222,152 +3329,6 @@ "jsep": "^0.4.0||^1.0.0" } }, - "node_modules/@langchain/core": { - "version": "0.3.61", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.61.tgz", - "integrity": "sha512-4O7fw5SXNSE+uBnathLQrhm3t+7dZGagt/5kt37A+pXw0AkudxEBvveg73sSnpBd9SIz3/Vc7F4k8rCKXGbEDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cfworker/json-schema": "^4.0.2", - "ansi-styles": "^5.0.0", - "camelcase": "6", - "decamelize": "1.2.0", - "js-tiktoken": "^1.0.12", - "langsmith": "^0.3.33", - "mustache": "^4.2.0", - "p-queue": "^6.6.2", - "p-retry": "4", - "uuid": "^10.0.0", - "zod": "^3.25.32", - "zod-to-json-schema": "^3.22.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@langchain/core/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@langchain/core/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@langchain/core/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@langchain/google-genai": { - "version": "0.2.14", - "resolved": "https://registry.npmjs.org/@langchain/google-genai/-/google-genai-0.2.14.tgz", - "integrity": "sha512-gKe/T2LNh8wSSMJOaFmYd8cwQnDSXKtVtC6a7CFoq5nWuh0bKzhItM/7bue1aMN8mlKfB2G1HCwxhaZoSpS/DA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@google/generative-ai": "^0.24.0", - "uuid": "^11.1.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.3.58 <0.4.0" - } - }, - "node_modules/@langchain/ollama": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@langchain/ollama/-/ollama-0.2.3.tgz", - "integrity": "sha512-1Obe45jgQspqLMBVlayQbGdywFmri8DgmGRdzNu0li56cG5RReYlRCFVDZBRMMvF9JhsP5eXRyfyivtKfITHWQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ollama": "^0.5.12", - "uuid": "^10.0.0" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.3.58 <0.4.0" - } - }, - "node_modules/@langchain/ollama/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@langchain/openai": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.5.16.tgz", - "integrity": "sha512-TqzPE3PM0bMkQi53qs8vCFkwaEp3VgwGw+s1e8Nas5ICCZZtc2XqcDPz4hf2gpo1k7/AZd6HuPlAsDy6wye9Qw==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tiktoken": "^1.0.12", - "openai": "^5.3.0", - "zod": "^3.25.32" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.3.58 <0.4.0" - } - }, - "node_modules/@langchain/textsplitters": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.1.0.tgz", - "integrity": "sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tiktoken": "^1.0.12" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/core": ">=0.2.21 <0.4.0" - } - }, "node_modules/@modelcontextprotocol/inspector": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/inspector/-/inspector-0.14.1.tgz", @@ -6382,6 +6343,13 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz", @@ -6454,13 +6422,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/retry": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", - "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/simple-oauth2": { "version": "5.0.7", "resolved": "https://registry.npmjs.org/@types/simple-oauth2/-/simple-oauth2-5.0.7.tgz", @@ -6490,13 +6451,6 @@ "license": "MIT", "optional": true }, - "node_modules/@types/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", - "dev": true, - "license": "MIT" - }, "node_modules/@types/webidl-conversions": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", @@ -6854,6 +6808,33 @@ "node": ">= 14" } }, + "node_modules/ai": { + "version": "4.3.16", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.16.tgz", + "integrity": "sha512-KUDwlThJ5tr2Vw0A1ZkbDKNME3wzWhuVfAOwIvFUzl1TPVDFAXDFTXio3p+jaKneB+dKNCvFFlolYmmgHttG1g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/ajv": { "version": "8.17.1", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", @@ -7943,16 +7924,6 @@ "node": ">=12" } }, - "node_modules/console-table-printer": { - "version": "2.14.6", - "resolved": "https://registry.npmjs.org/console-table-printer/-/console-table-printer-2.14.6.tgz", - "integrity": "sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw==", - "dev": true, - "license": "MIT", - "dependencies": { - "simple-wcswidth": "^1.0.1" - } - }, "node_modules/content-disposition": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.0.tgz", @@ -8143,16 +8114,6 @@ } } }, - "node_modules/decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/decko": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decko/-/decko-1.2.0.tgz", @@ -8576,6 +8537,16 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -8623,6 +8594,13 @@ "node": ">=0.3.1" } }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "dev": true, + "license": "Apache-2.0" + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -11300,16 +11278,6 @@ "node": ">=0.10.0" } }, - "node_modules/js-tiktoken": { - "version": "1.0.20", - "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.20.tgz", - "integrity": "sha512-Xlaqhhs8VfCd6Sh7a1cFkZHQbYTLCwVJJWiHVxBYzLPxW0XsoxBy1hitmjkdIjD3Aon5BXLHFwU5O8WUx6HH+A==", - "dev": true, - "license": "MIT", - "dependencies": { - "base64-js": "^1.5.1" - } - }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -11383,6 +11351,13 @@ "foreach": "^2.0.4" } }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "dev": true, + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -11410,6 +11385,37 @@ "node": ">=6" } }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/jsondiffpatch/node_modules/chalk": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz", + "integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/jsonpath-plus": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-10.3.0.tgz", @@ -11474,154 +11480,6 @@ "node": ">=6" } }, - "node_modules/langchain": { - "version": "0.3.29", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.3.29.tgz", - "integrity": "sha512-L389pKlApVJPqu4hp58qY6NZAobI+MFPoBjSfjT1z3mcxtB68wLFGhaH4DVsTVg21NYO+0wTEoz24BWrxu9YGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@langchain/openai": ">=0.1.0 <0.6.0", - "@langchain/textsplitters": ">=0.0.0 <0.2.0", - "js-tiktoken": "^1.0.12", - "js-yaml": "^4.1.0", - "jsonpointer": "^5.0.1", - "langsmith": "^0.3.33", - "openapi-types": "^12.1.3", - "p-retry": "4", - "uuid": "^10.0.0", - "yaml": "^2.2.1", - "zod": "^3.25.32" - }, - "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@langchain/anthropic": "*", - "@langchain/aws": "*", - "@langchain/cerebras": "*", - "@langchain/cohere": "*", - "@langchain/core": ">=0.3.58 <0.4.0", - "@langchain/deepseek": "*", - "@langchain/google-genai": "*", - "@langchain/google-vertexai": "*", - "@langchain/google-vertexai-web": "*", - "@langchain/groq": "*", - "@langchain/mistralai": "*", - "@langchain/ollama": "*", - "@langchain/xai": "*", - "axios": "*", - "cheerio": "*", - "handlebars": "^4.7.8", - "peggy": "^3.0.2", - "typeorm": "*" - }, - "peerDependenciesMeta": { - "@langchain/anthropic": { - "optional": true - }, - "@langchain/aws": { - "optional": true - }, - "@langchain/cerebras": { - "optional": true - }, - "@langchain/cohere": { - "optional": true - }, - "@langchain/deepseek": { - "optional": true - }, - "@langchain/google-genai": { - "optional": true - }, - "@langchain/google-vertexai": { - "optional": true - }, - "@langchain/google-vertexai-web": { - "optional": true - }, - "@langchain/groq": { - "optional": true - }, - "@langchain/mistralai": { - "optional": true - }, - "@langchain/ollama": { - "optional": true - }, - "@langchain/xai": { - "optional": true - }, - "axios": { - "optional": true - }, - "cheerio": { - "optional": true - }, - "handlebars": { - "optional": true - }, - "peggy": { - "optional": true - }, - "typeorm": { - "optional": true - } - } - }, - "node_modules/langchain/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/langsmith": { - "version": "0.3.34", - "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.3.34.tgz", - "integrity": "sha512-rxYuuypqaSzIuNjZMTsCVAgG0cYdI516dwuKn58bu4YuBRlLaLeNlHewRyoqP9lrEAlpkekCV9fUwwZ7lO8f2g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/uuid": "^10.0.0", - "chalk": "^4.1.2", - "console-table-printer": "^2.12.1", - "p-queue": "^6.6.2", - "p-retry": "4", - "semver": "^7.6.3", - "uuid": "^10.0.0" - }, - "peerDependencies": { - "openai": "*" - }, - "peerDependenciesMeta": { - "openai": { - "optional": true - } - } - }, - "node_modules/langsmith/node_modules/uuid": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", - "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist/bin/uuid" - } - }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -12392,16 +12250,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, - "node_modules/mustache": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", - "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", - "dev": true, - "license": "MIT", - "bin": { - "mustache": "bin/mustache" - } - }, "node_modules/nan": { "version": "2.22.2", "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", @@ -12744,14 +12592,27 @@ "node": "^10.13.0 || >=12.0.0" } }, - "node_modules/ollama": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.16.tgz", - "integrity": "sha512-OEbxxOIUZtdZgOaTPAULo051F5y+Z1vosxEYOoABPnQKeW7i4O8tJNlxCB+xioyoorVqgjkdj+TA1f1Hy2ug/w==", + "node_modules/ollama-ai-provider": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ollama-ai-provider/-/ollama-ai-provider-1.2.0.tgz", + "integrity": "sha512-jTNFruwe3O/ruJeppI/quoOUxG7NA6blG3ZyQj3lei4+NnJo7bi3eIRWqlVpRlu/mbzbFXeJSBuYQWF6pzGKww==", "dev": true, - "license": "MIT", + "license": "Apache-2.0", "dependencies": { - "whatwg-fetch": "^3.6.20" + "@ai-sdk/provider": "^1.0.0", + "@ai-sdk/provider-utils": "^2.0.0", + "partial-json": "0.1.7" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } } }, "node_modules/on-finished": { @@ -12808,28 +12669,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/openai": { - "version": "5.8.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.8.2.tgz", - "integrity": "sha512-8C+nzoHYgyYOXhHGN6r0fcb4SznuEn1R7YZMvlqDbnCuE0FM2mm3T1HiYW6WIcMS/F1Of2up/cSPjLPaWt0X9Q==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "openai": "bin/cli" - }, - "peerDependencies": { - "ws": "^8.18.0", - "zod": "^3.23.8" - }, - "peerDependenciesMeta": { - "ws": { - "optional": true - }, - "zod": { - "optional": true - } - } - }, "node_modules/openapi-fetch": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.14.0.tgz", @@ -13031,16 +12870,6 @@ "dev": true, "license": "MIT" }, - "node_modules/p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -13073,57 +12902,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-queue": { - "version": "6.6.2", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", - "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "eventemitter3": "^4.0.4", - "p-timeout": "^3.2.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-queue/node_modules/eventemitter3": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", - "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", - "dev": true, - "license": "MIT" - }, - "node_modules/p-retry": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", - "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/retry": "0.12.0", - "retry": "^0.13.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-timeout": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", - "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-finally": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -13207,6 +12985,13 @@ "node": ">= 0.8" } }, + "node_modules/partial-json": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/partial-json/-/partial-json-0.1.7.tgz", + "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", + "dev": true, + "license": "MIT" + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", @@ -14129,16 +13914,6 @@ "node": ">=10" } }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } - }, "node_modules/reusify": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", @@ -14251,6 +14026,13 @@ "loose-envify": "^1.1.0" } }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "dev": true, + "license": "BSD-3-Clause" + }, "node_modules/seek-bzip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", @@ -14673,13 +14455,6 @@ "joi": "^17.6.4" } }, - "node_modules/simple-wcswidth": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/simple-wcswidth/-/simple-wcswidth-1.1.2.tgz", - "integrity": "sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==", - "dev": true, - "license": "MIT" - }, "node_modules/simple-websocket": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/simple-websocket/-/simple-websocket-9.1.0.tgz", @@ -15179,6 +14954,20 @@ "node": ">= 6" } }, + "node_modules/swr": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.3.3.tgz", + "integrity": "sha512-dshNvs3ExOqtZ6kJBaAsabhPdHyeY4P2cKwRCniDVifBMoG/SVI7tfLWqPXriVspf2Rg4tPzXJTnwaihIeFw2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/synckit": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.8.tgz", @@ -15392,6 +15181,19 @@ "node": "*" } }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -15999,13 +15801,6 @@ "node": ">=12" } }, - "node_modules/whatwg-fetch": { - "version": "3.6.20", - "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", - "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==", - "dev": true, - "license": "MIT" - }, "node_modules/whatwg-url": { "version": "14.2.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-14.2.0.tgz", diff --git a/package.json b/package.json index 9fb6a9b5..bd8af4a8 100644 --- a/package.json +++ b/package.json @@ -34,17 +34,15 @@ "license": "Apache-2.0", "devDependencies": { "@eslint/js": "^9.24.0", + "@himanshusinghs/google": "^1.2.11", "@jest/globals": "^30.0.0", - "@langchain/core": "^0.3.61", - "@langchain/google-genai": "^0.2.14", - "@langchain/ollama": "^0.2.3", - "@langchain/openai": "^0.5.16", "@modelcontextprotocol/inspector": "^0.14.0", "@redocly/cli": "^1.34.2", "@types/jest": "^29.5.14", "@types/node": "^22.14.0", "@types/simple-oauth2": "^5.0.7", "@types/yargs-parser": "^21.0.3", + "ai": "^4.3.16", "eslint": "^9.24.0", "eslint-config-prettier": "^10.1.2", "eslint-plugin-jest": "^29.0.1", @@ -53,8 +51,9 @@ "jest": "^29.7.0", "jest-environment-node": "^29.7.0", "jest-extended": "^6.0.0", - "langchain": "^0.3.29", + "json-schema": "^0.4.0", "mongodb-runner": "^5.8.2", + "ollama-ai-provider": "^1.2.0", "openapi-types": "^12.1.3", "openapi-typescript": "^7.6.1", "prettier": "^3.5.3", diff --git a/tests/accuracy/sdk/agent.ts b/tests/accuracy/sdk/agent.ts new file mode 100644 index 00000000..905cfff9 --- /dev/null +++ b/tests/accuracy/sdk/agent.ts @@ -0,0 +1,38 @@ +import { generateText, Tool, Schema, LanguageModelV1 } from "ai"; +import { Model } from "./models.js"; + +const systemPrompt = [ + 'The keywords "MUST", "MUST NOT", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119', + "You are an expert AI assistant with access to a set of tools for MongoDB database operations.", + "You MUST use the most relevant tool to answer the user's request", + "When calling a tool, you MUST strictly follow its input schema and MUST provide all required arguments", + "If a task requires multiple steps, you MUST call the necessary tools in sequence", + 'If you do not know the answer or the request cannot be fulfilled, you MUST reply with "I don\'t know"', + "You SHOULD assume that you are already connected to a MongoDB connection", +].join("\n"); + +export interface Agent { + prompt(prompt: string, model: M, tools: T): Promise; +} + +export function getVercelToolCallingAgent(): Agent< + Model, + Record>>, + { text: string; messages: unknown[] } +> { + return { + async prompt(prompt: string, model: Model, tools: Record>>) { + const result = await generateText({ + model: model.getModel(), + system: systemPrompt, + prompt, + tools, + maxSteps: 100, + }); + return { + text: result.text, + messages: result.response.messages, + }; + }, + }; +} diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index 0ec4bb64..97496f6e 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -1,14 +1,13 @@ -import { AgentExecutor } from "langchain/agents"; import { Tool } from "@modelcontextprotocol/sdk/types.js"; -import { discoverMongoDBTools, TestTools, ToolResultGenerators } from "./test-tools.js"; +import { discoverMongoDBTools, TestTools, MockedTools } from "./test-tools.js"; import { TestableModels } from "./models.js"; -import { getToolCallingAgent } from "./tool-calling-agent.js"; import { ExpectedToolCall, parameterMatchingAccuracyScorer, toolCallingAccuracyScorer } from "./accuracy-scorers.js"; +import { Agent, getVercelToolCallingAgent } from "./agent.js"; interface AccuracyTestConfig { prompt: string; expectedToolCalls: ExpectedToolCall[]; - mockedTools: ToolResultGenerators; + mockedTools: MockedTools; } export function describeAccuracyTests( @@ -22,7 +21,7 @@ export function describeAccuracyTests( eachModel(`$modelName - ${suiteName}`, function (model) { let mcpTools: Tool[]; let testTools: TestTools; - let agent: AgentExecutor; + let agent: Agent; beforeAll(async () => { mcpTools = await discoverMongoDBTools(); @@ -30,13 +29,12 @@ export function describeAccuracyTests( beforeEach(() => { testTools = new TestTools(mcpTools); - const transformToolResult = model.transformToolResult.bind(model); - agent = getToolCallingAgent(model, testTools.langChainTools(transformToolResult)); + agent = getVercelToolCallingAgent(); }); eachTest("$prompt", async function (testConfig) { testTools.mockTools(testConfig.mockedTools); - const conversation = await agent.invoke({ input: testConfig.prompt }); + const conversation = await agent.prompt(testConfig.prompt, model, testTools.vercelAiTools()); console.log("conversation", conversation); const toolCalls = testTools.getToolCalls(); console.log("?????? toolCalls", toolCalls); diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts index d370633f..832aad30 100644 --- a/tests/accuracy/sdk/models.ts +++ b/tests/accuracy/sdk/models.ts @@ -1,58 +1,42 @@ -import { BaseChatModel } from "@langchain/core/language_models/chat_models"; -import { ChatGoogleGenerativeAI } from "@langchain/google-genai"; -import { ChatOllama } from "@langchain/ollama"; -import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; +import { LanguageModelV1 } from "ai"; +import { createGoogleGenerativeAI } from "@himanshusinghs/google"; +import { ollama } from "ollama-ai-provider"; -type ToolResultForOllama = string; -export type AcceptableToolResponse = CallToolResult | ToolResultForOllama; - -export interface Model { +export interface Model

{ isAvailable(): boolean; - getLangChainModel(): M; - transformToolResult(callToolResult: CallToolResult): T; + getModel(): P; } -export class GeminiModel implements Model { +export class GeminiModel implements Model { constructor(readonly modelName: string) {} isAvailable(): boolean { return !!process.env.MDB_GEMINI_API_KEY; } - getLangChainModel(): ChatGoogleGenerativeAI { - return new ChatGoogleGenerativeAI({ - model: this.modelName, + getModel() { + return createGoogleGenerativeAI({ apiKey: process.env.MDB_GEMINI_API_KEY, - }); - } - - transformToolResult(callToolResult: CallToolResult) { - return callToolResult; + })(this.modelName); } } -export class OllamaModel implements Model { +export class OllamaModel implements Model { constructor(readonly modelName: string) {} isAvailable(): boolean { - return !!process.env.MDB_GEMINI_API_KEY; - } - - getLangChainModel(): ChatOllama { - return new ChatOllama({ - model: this.modelName, - }); + return true; } - transformToolResult(callToolResult: CallToolResult): ToolResultForOllama { - return JSON.stringify(callToolResult); + getModel() { + return ollama(this.modelName); } } const ALL_TESTABLE_MODELS = [ - // new GeminiModel("gemini-1.5-flash"), + new GeminiModel("gemini-1.5-flash"), // new GeminiModel("gemini-2.0-flash"), - new OllamaModel("qwen3:latest"), + // new OllamaModel("qwen3:latest"), ]; export type TestableModels = ReturnType; diff --git a/tests/accuracy/sdk/test-tools.ts b/tests/accuracy/sdk/test-tools.ts index 82719454..cb728a36 100644 --- a/tests/accuracy/sdk/test-tools.ts +++ b/tests/accuracy/sdk/test-tools.ts @@ -1,6 +1,6 @@ -import { jest } from "@jest/globals"; +import { JSONSchema7 } from "json-schema"; import { v4 as uuid } from "uuid"; -import { DynamicTool, tool as langChainTool } from "@langchain/core/tools"; +import { Tool as VercelTool, Schema, tool as createVercelTool, jsonSchema } from "ai"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { CallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js"; @@ -10,15 +10,22 @@ import { defaultTestConfig } from "../../integration/helpers.js"; import { Session } from "../../../src/session.js"; import { Telemetry } from "../../../src/telemetry/telemetry.js"; import { Server } from "../../../src/server.js"; -import { AcceptableToolResponse } from "./models.js"; import { ToolCall } from "./accuracy-scorers.js"; type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult; -type MockedToolResultGeneratorFn = jest.MockedFunction; -type MockedTools = Record; -export type ToolResultGenerators = Record; -export type LangChainTool = DynamicTool; -export type ToolResultTransformer = (toolResult: CallToolResult) => T; +export type MockedTools = Record; + +function getDefaultToolResultGeneratorFn(): ToolResultGeneratorFn { + return () => ({ + content: [ + { + type: "text", + text: `Mock implementation for tool not present`, + }, + ], + isError: true, + }); +} export class TestTools { private mockedTools: MockedTools = {}; @@ -26,15 +33,7 @@ export class TestTools { constructor(private readonly mcpTools: Tool[]) { for (const mcpTool of mcpTools) { - this.mockedTools[mcpTool.name] = jest.fn().mockReturnValue({ - content: [ - { - type: "text", - text: `Mock implementation for tool - ${mcpTool.name} not present`, - }, - ], - isError: true, - }); + this.mockedTools[mcpTool.name] = getDefaultToolResultGeneratorFn(); } } @@ -42,9 +41,9 @@ export class TestTools { return this.recordedToolCalls; } - mockTools(toolResultGenerators: ToolResultGenerators) { - for (const toolName in toolResultGenerators) { - const toolResultGeneratorFn = toolResultGenerators[toolName]; + mockTools(mockedTools: MockedTools) { + for (const toolName in mockedTools) { + const toolResultGeneratorFn = mockedTools[toolName]; if (!this.mockedTools[toolName]) { throw new Error(`Attempted to mock unrecognized tool - ${toolName}`); } @@ -53,49 +52,40 @@ export class TestTools { // Are you happy TS? continue; } - this.mockedTools[toolName] = jest.fn(toolResultGeneratorFn); + this.mockedTools[toolName] = toolResultGeneratorFn; } } - langChainTools( - transformToolResult: ToolResultTransformer - ): LangChainTool[] { - return this.mcpTools.map((mcpTool) => { - return langChainTool((...args) => { - console.log("????? args", args); - const [parameters, { runName, runId }] = args; - const toolCallId = typeof runId !== "undefined" ? `${runId}` : uuid(); - return this.langChainToolResultGenerator(`${runName}`, parameters, toolCallId, transformToolResult); - }, mcpTool); - }); - } - - private langChainToolResultGenerator( - tool: string, - parameters: unknown, - toolCallId: string, - transformToolResult: ToolResultTransformer - ): T { - this.recordedToolCalls.push({ - toolCallId: toolCallId, - toolName: tool, - parameters, - }); - const mockedToolResultGenerator = this.mockedTools[tool]; - if (!mockedToolResultGenerator) { - // log as well - return transformToolResult({ - content: [ - { - type: "text", - text: `Could not resolve tool generator for ${tool}`, - }, - ], - isError: true, + vercelAiTools(): Record>> { + const vercelTools: Record>> = {}; + for (const tool of this.mcpTools) { + vercelTools[tool.name] = createVercelTool({ + description: tool.description, + parameters: jsonSchema(tool.inputSchema as JSONSchema7), + // eslint-disable-next-line @typescript-eslint/require-await + execute: async (args: unknown) => { + this.recordedToolCalls.push({ + toolCallId: uuid(), + toolName: tool.name, + parameters: args, + }); + const toolResultGeneratorFn = this.mockedTools[tool.name]; + if (!toolResultGeneratorFn) { + return { + content: [ + { + type: "text", + text: `Could not resolve tool generator for ${tool.name}`, + }, + ], + }; + } + + return toolResultGeneratorFn(args); + }, }); } - - return transformToolResult(mockedToolResultGenerator(parameters)); + return vercelTools; } } diff --git a/tests/accuracy/sdk/tool-calling-agent.ts b/tests/accuracy/sdk/tool-calling-agent.ts deleted file mode 100644 index b9adedf5..00000000 --- a/tests/accuracy/sdk/tool-calling-agent.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { ChatPromptTemplate } from "@langchain/core/prompts"; -import { createToolCallingAgent, AgentExecutor } from "langchain/agents"; - -import { LangChainTool } from "./test-tools.js"; -import { AcceptableToolResponse, Model } from "./models.js"; -import { BaseChatModel } from "@langchain/core/language_models/chat_models"; - -const prompt = ChatPromptTemplate.fromMessages([ - [ - "system", - [ - 'The keywords "MUST", "MUST NOT", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in RFC 2119 (https://www.ietf.org/rfc/rfc2119.txt)', - "You are an expect AI assistant with access to a set of tools for MongoDB database operations.", - "You MUST use the most relevant tool to answer the user's request", - "When calling a tool, you MUST strictly follow its input schema and MUST provide all required arguments", - "If a task requires multiple steps, you MUST call the necessary tools in sequence", - 'If you do not know the answer or the request cannot be fulfilled, you MUST reply with "I don\'t know"', - ].join("\n"), - ], - ["human", "{input}"], - ["placeholder", "{agent_scratchpad}"], -]); - -export function getToolCallingAgent( - model: Model, - tools: LangChainTool[] -) { - const llm = model.getLangChainModel(); - const agent = createToolCallingAgent({ - llm, - tools, - prompt, - }); - const agentExecutor = new AgentExecutor({ agent, tools }); - return agentExecutor; -} From 074c3cc49599e6e6fc117f210610716bf2dbf41d Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 30 Jun 2025 13:01:57 +0200 Subject: [PATCH 03/16] chore: integrate capturing accuracy snapshots --- package.json | 3 +- tests/accuracy/sdk/accuracy-snapshot.ts | 54 +++++++++++++++++++ tests/accuracy/sdk/describe-accuracy-tests.ts | 40 ++++++++++++-- tests/accuracy/sdk/models.ts | 7 +-- tests/accuracy/sdk/test-tools.ts | 3 -- 5 files changed, 95 insertions(+), 12 deletions(-) create mode 100644 tests/accuracy/sdk/accuracy-snapshot.ts diff --git a/package.json b/package.json index bd8af4a8..6eab1d11 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ "check:types": "tsc --noEmit --project tsconfig.json", "reformat": "prettier --write .", "generate": "./scripts/generate.sh", - "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage" + "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathIgnorePatterns=/tests/accuracy/", + "test:accuracy": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern tests/accuracy" }, "license": "Apache-2.0", "devDependencies": { diff --git a/tests/accuracy/sdk/accuracy-snapshot.ts b/tests/accuracy/sdk/accuracy-snapshot.ts new file mode 100644 index 00000000..1f7867a9 --- /dev/null +++ b/tests/accuracy/sdk/accuracy-snapshot.ts @@ -0,0 +1,54 @@ +import fs from "fs/promises"; +import path from "path"; +import { z } from "zod"; + +export const SNAPSHOT_FILE_PATH = path.resolve(process.cwd(), "accuracy-snapshot.json"); + +export const AccuracySnapshotEntrySchema = z.object({ + datetime: z.string(), + commit: z.string(), + model: z.string(), + suite: z.string(), + test: z.string(), + toolCallingAccuracy: z.number(), + parameterAccuracy: z.number(), +}); + +export type AccuracySnapshotEntry = z.infer; + +export async function readSnapshot(): Promise { + try { + const raw = await fs.readFile(SNAPSHOT_FILE_PATH, "utf8"); + return AccuracySnapshotEntrySchema.array().parse(JSON.parse(raw)); + } catch (e: unknown) { + if ((e as { code: string }).code === "ENOENT") { + return []; + } + throw e; + } +} + +function waitFor(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function appendAccuracySnapshot(entry: AccuracySnapshotEntry): Promise { + AccuracySnapshotEntrySchema.parse(entry); + + for (let attempt = 0; attempt < 5; attempt++) { + try { + const snapshot = await readSnapshot(); + snapshot.unshift(entry); + const tmp = `${SNAPSHOT_FILE_PATH}~${Date.now()}`; + await fs.writeFile(tmp, JSON.stringify(snapshot, null, 2)); + await fs.rename(tmp, SNAPSHOT_FILE_PATH); + return; + } catch (e) { + if (attempt < 4) { + await waitFor(100 + Math.random() * 200); + } else { + throw e; + } + } + } +} diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index 97496f6e..a3ad0668 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -3,6 +3,7 @@ import { discoverMongoDBTools, TestTools, MockedTools } from "./test-tools.js"; import { TestableModels } from "./models.js"; import { ExpectedToolCall, parameterMatchingAccuracyScorer, toolCallingAccuracyScorer } from "./accuracy-scorers.js"; import { Agent, getVercelToolCallingAgent } from "./agent.js"; +import { appendAccuracySnapshot } from "./accuracy-snapshot.js"; interface AccuracyTestConfig { prompt: string; @@ -15,6 +16,20 @@ export function describeAccuracyTests( models: TestableModels, accuracyTestConfigs: AccuracyTestConfig[] ) { + const accuracyDatetime = process.env.ACCURACY_DATETIME; + if (!accuracyDatetime) { + throw new Error("ACCURACY_DATETIME environment variable is not set"); + } + const accuracyCommit = process.env.ACCURACY_COMMIT; + if (!accuracyCommit) { + throw new Error("ACCURACY_COMMIT environment variable is not set"); + } + + if (!models.length) { + console.warn(`No models available to test ${suiteName}`); + return; + } + const eachModel = describe.each(models); const eachTest = it.each(accuracyTestConfigs); @@ -35,15 +50,30 @@ export function describeAccuracyTests( eachTest("$prompt", async function (testConfig) { testTools.mockTools(testConfig.mockedTools); const conversation = await agent.prompt(testConfig.prompt, model, testTools.vercelAiTools()); - console.log("conversation", conversation); const toolCalls = testTools.getToolCalls(); - console.log("?????? toolCalls", toolCalls); - console.log("???? expected", testConfig.expectedToolCalls); const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); + await appendAccuracySnapshot({ + datetime: accuracyDatetime, + commit: accuracyCommit, + model: model.modelName, + suite: suiteName, + test: testConfig.prompt, + toolCallingAccuracy, + parameterAccuracy: parameterMatchingAccuracy, + }); - expect(toolCallingAccuracy).not.toEqual(0); - expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); + try { + expect(toolCallingAccuracy).not.toEqual(0); + expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); + } catch (error) { + console.warn(`Accuracy test failed for ${model.modelName} - ${suiteName} - ${testConfig.prompt}`); + console.warn(`Conversation`, JSON.stringify(conversation, null, 2)); + console.warn(`Tool calls`, JSON.stringify(toolCalls, null, 2)); + console.warn(`Tool calling accuracy`, toolCallingAccuracy); + console.warn(`Parameter matching accuracy`, parameterMatchingAccuracy); + throw error; + } }); }); } diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts index 832aad30..27b8e972 100644 --- a/tests/accuracy/sdk/models.ts +++ b/tests/accuracy/sdk/models.ts @@ -3,6 +3,7 @@ import { createGoogleGenerativeAI } from "@himanshusinghs/google"; import { ollama } from "ollama-ai-provider"; export interface Model

{ + readonly modelName: string; isAvailable(): boolean; getModel(): P; } @@ -25,7 +26,7 @@ export class OllamaModel implements Model { constructor(readonly modelName: string) {} isAvailable(): boolean { - return true; + return false; } getModel() { @@ -35,8 +36,8 @@ export class OllamaModel implements Model { const ALL_TESTABLE_MODELS = [ new GeminiModel("gemini-1.5-flash"), - // new GeminiModel("gemini-2.0-flash"), - // new OllamaModel("qwen3:latest"), + new GeminiModel("gemini-2.0-flash"), + new OllamaModel("qwen3:latest"), ]; export type TestableModels = ReturnType; diff --git a/tests/accuracy/sdk/test-tools.ts b/tests/accuracy/sdk/test-tools.ts index cb728a36..595a9069 100644 --- a/tests/accuracy/sdk/test-tools.ts +++ b/tests/accuracy/sdk/test-tools.ts @@ -132,9 +132,6 @@ export async function discoverMongoDBTools(): Promise { await mcpClient.connect(clientTransport); return (await mcpClient.listTools()).tools; - } catch (error: unknown) { - console.error("Unexpected error occured", error); - return []; } finally { await mcpClient?.close(); await mcpServer?.session?.close(); From be684e6c4e1f824c21fef01311eb11b9dd82ee14 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 30 Jun 2025 13:03:50 +0200 Subject: [PATCH 04/16] chore: correct env names --- tests/accuracy/sdk/describe-accuracy-tests.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index a3ad0668..5d500ffa 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -16,13 +16,13 @@ export function describeAccuracyTests( models: TestableModels, accuracyTestConfigs: AccuracyTestConfig[] ) { - const accuracyDatetime = process.env.ACCURACY_DATETIME; + const accuracyDatetime = process.env.MDB_ACCURACY_DATETIME; if (!accuracyDatetime) { - throw new Error("ACCURACY_DATETIME environment variable is not set"); + throw new Error("MDB_ACCURACY_DATETIME environment variable is not set"); } - const accuracyCommit = process.env.ACCURACY_COMMIT; + const accuracyCommit = process.env.MDB_ACCURACY_COMMIT; if (!accuracyCommit) { - throw new Error("ACCURACY_COMMIT environment variable is not set"); + throw new Error("MDB_ACCURACY_COMMIT environment variable is not set"); } if (!models.length) { From e8f6ad35f1df158e01e3601b6af48b7cd5f181ff Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 30 Jun 2025 13:29:32 +0200 Subject: [PATCH 05/16] chore: more consolidated prompt tests --- tests/accuracy/list-databases.test.ts | 28 +++++++++---- tests/accuracy/sdk/agent.ts | 13 +++--- tests/accuracy/sdk/describe-accuracy-tests.ts | 41 ++++++++++--------- tests/accuracy/sdk/models.ts | 4 +- 4 files changed, 48 insertions(+), 38 deletions(-) diff --git a/tests/accuracy/list-databases.test.ts b/tests/accuracy/list-databases.test.ts index ae3f6c7d..d26fbc4e 100644 --- a/tests/accuracy/list-databases.test.ts +++ b/tests/accuracy/list-databases.test.ts @@ -1,16 +1,22 @@ import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; import { getAvailableModels } from "./sdk/models.js"; +import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; -describeAccuracyTests("list-databases", getAvailableModels(), [ - { - prompt: "Assume that you're already connected. How many collections are there in sample_mflix database", +function describeListDatabasesAccuracyTests(prompt: string): AccuracyTestConfig { + return { + systemPrompt: "Assume that you're already connected.", + prompt: prompt, mockedTools: { - "list-collections": function listCollections() { + "list-databases": function listDatabases() { return { content: [ { type: "text", - text: "Name: coll1", + text: "Name: db1", + }, + { + type: "text", + text: "Name: db2", }, ], }; @@ -18,9 +24,15 @@ describeAccuracyTests("list-databases", getAvailableModels(), [ }, expectedToolCalls: [ { - toolName: "list-collections", - parameters: { database: "sample_mflix" }, + toolName: "list-databases", + parameters: {}, }, ], - }, + }; +} + +describeAccuracyTests("list-databases", getAvailableModels(), [ + describeListDatabasesAccuracyTests("How many databases do I have?"), + describeListDatabasesAccuracyTests("List all the databases in my cluster."), + describeListDatabasesAccuracyTests("Is there a sample_mflix database in my cluster?"), ]); diff --git a/tests/accuracy/sdk/agent.ts b/tests/accuracy/sdk/agent.ts index 905cfff9..eb680358 100644 --- a/tests/accuracy/sdk/agent.ts +++ b/tests/accuracy/sdk/agent.ts @@ -8,23 +8,20 @@ const systemPrompt = [ "When calling a tool, you MUST strictly follow its input schema and MUST provide all required arguments", "If a task requires multiple steps, you MUST call the necessary tools in sequence", 'If you do not know the answer or the request cannot be fulfilled, you MUST reply with "I don\'t know"', - "You SHOULD assume that you are already connected to a MongoDB connection", -].join("\n"); +]; export interface Agent { prompt(prompt: string, model: M, tools: T): Promise; } -export function getVercelToolCallingAgent(): Agent< - Model, - Record>>, - { text: string; messages: unknown[] } -> { +export function getVercelToolCallingAgent( + requestedSystemPrompt?: string +): Agent, Record>>, { text: string; messages: unknown[] }> { return { async prompt(prompt: string, model: Model, tools: Record>>) { const result = await generateText({ model: model.getModel(), - system: systemPrompt, + system: [...systemPrompt, requestedSystemPrompt].join("\n"), prompt, tools, maxSteps: 100, diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index 5d500ffa..bf99d509 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -5,7 +5,8 @@ import { ExpectedToolCall, parameterMatchingAccuracyScorer, toolCallingAccuracyS import { Agent, getVercelToolCallingAgent } from "./agent.js"; import { appendAccuracySnapshot } from "./accuracy-snapshot.js"; -interface AccuracyTestConfig { +export interface AccuracyTestConfig { + systemPrompt?: string; prompt: string; expectedToolCalls: ExpectedToolCall[]; mockedTools: MockedTools; @@ -17,13 +18,7 @@ export function describeAccuracyTests( accuracyTestConfigs: AccuracyTestConfig[] ) { const accuracyDatetime = process.env.MDB_ACCURACY_DATETIME; - if (!accuracyDatetime) { - throw new Error("MDB_ACCURACY_DATETIME environment variable is not set"); - } const accuracyCommit = process.env.MDB_ACCURACY_COMMIT; - if (!accuracyCommit) { - throw new Error("MDB_ACCURACY_COMMIT environment variable is not set"); - } if (!models.length) { console.warn(`No models available to test ${suiteName}`); @@ -53,25 +48,31 @@ export function describeAccuracyTests( const toolCalls = testTools.getToolCalls(); const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); - await appendAccuracySnapshot({ - datetime: accuracyDatetime, - commit: accuracyCommit, - model: model.modelName, - suite: suiteName, - test: testConfig.prompt, - toolCallingAccuracy, - parameterAccuracy: parameterMatchingAccuracy, - }); + if (accuracyDatetime && accuracyCommit) { + await appendAccuracySnapshot({ + datetime: accuracyDatetime, + commit: accuracyCommit, + model: model.modelName, + suite: suiteName, + test: testConfig.prompt, + toolCallingAccuracy, + parameterAccuracy: parameterMatchingAccuracy, + }); + } else { + console.info( + `Skipping accuracy snapshot update for ${model.modelName} - ${suiteName} - ${testConfig.prompt}` + ); + } try { expect(toolCallingAccuracy).not.toEqual(0); expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); } catch (error) { console.warn(`Accuracy test failed for ${model.modelName} - ${suiteName} - ${testConfig.prompt}`); - console.warn(`Conversation`, JSON.stringify(conversation, null, 2)); - console.warn(`Tool calls`, JSON.stringify(toolCalls, null, 2)); - console.warn(`Tool calling accuracy`, toolCallingAccuracy); - console.warn(`Parameter matching accuracy`, parameterMatchingAccuracy); + console.debug(`Conversation`, JSON.stringify(conversation, null, 2)); + console.debug(`Tool calls`, JSON.stringify(toolCalls, null, 2)); + console.debug(`Tool calling accuracy`, toolCallingAccuracy); + console.debug(`Parameter matching accuracy`, parameterMatchingAccuracy); throw error; } }); diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts index 27b8e972..f6f8a879 100644 --- a/tests/accuracy/sdk/models.ts +++ b/tests/accuracy/sdk/models.ts @@ -36,8 +36,8 @@ export class OllamaModel implements Model { const ALL_TESTABLE_MODELS = [ new GeminiModel("gemini-1.5-flash"), - new GeminiModel("gemini-2.0-flash"), - new OllamaModel("qwen3:latest"), + // new GeminiModel("gemini-2.0-flash"), + // new OllamaModel("qwen3:latest"), ]; export type TestableModels = ReturnType; From 88641342eb30be4f014c0753ecc87c0f4f63f7c0 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Mon, 30 Jun 2025 16:37:10 +0200 Subject: [PATCH 06/16] chore: add a few more tests and some more models --- tests/accuracy/list-collections.test.ts | 38 +++++++++++++++++++++++++ tests/accuracy/sdk/models.ts | 6 ++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 tests/accuracy/list-collections.test.ts diff --git a/tests/accuracy/list-collections.test.ts b/tests/accuracy/list-collections.test.ts new file mode 100644 index 00000000..b871a96e --- /dev/null +++ b/tests/accuracy/list-collections.test.ts @@ -0,0 +1,38 @@ +import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; +import { getAvailableModels } from "./sdk/models.js"; +import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; + +function describeListCollectionsAccuracyTests(prompt: string): AccuracyTestConfig { + return { + systemPrompt: "Assume that you're already connected.", + prompt: prompt, + mockedTools: { + "list-collections": function listCollections() { + return { + content: [ + { + type: "text", + text: "Name: coll1", + }, + { + type: "text", + text: "Name: coll1", + }, + ], + }; + }, + }, + expectedToolCalls: [ + { + toolName: "list-collections", + parameters: { database: "db1" }, + }, + ], + }; +} + +describeAccuracyTests("list-collections", getAvailableModels(), [ + describeListCollectionsAccuracyTests("How many collections do I have in database db1?"), + describeListCollectionsAccuracyTests("List all the collections in my MongoDB database db1."), + describeListCollectionsAccuracyTests("Is there a coll1 collection in my MongoDB database db1?"), +]); diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts index f6f8a879..e3f5ab1f 100644 --- a/tests/accuracy/sdk/models.ts +++ b/tests/accuracy/sdk/models.ts @@ -26,7 +26,7 @@ export class OllamaModel implements Model { constructor(readonly modelName: string) {} isAvailable(): boolean { - return false; + return true; } getModel() { @@ -36,8 +36,8 @@ export class OllamaModel implements Model { const ALL_TESTABLE_MODELS = [ new GeminiModel("gemini-1.5-flash"), - // new GeminiModel("gemini-2.0-flash"), - // new OllamaModel("qwen3:latest"), + new GeminiModel("gemini-2.0-flash"), + new OllamaModel("qwen3:1.7b"), ]; export type TestableModels = ReturnType; From 96dc0ce170486ed9015e8b82741e49f0e7f6e8f0 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 13:01:41 +0200 Subject: [PATCH 07/16] chore: add AzureOpenAI model in the model list --- package-lock.json | 36 +++++++++++++++++++ package.json | 1 + tests/accuracy/list-collections.test.ts | 2 +- tests/accuracy/list-databases.test.ts | 2 +- tests/accuracy/sdk/describe-accuracy-tests.ts | 6 +++- tests/accuracy/sdk/models.ts | 21 +++++++++-- 6 files changed, 63 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 34e1c677..ecac99ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "mongodb-mcp-server": "dist/index.js" }, "devDependencies": { + "@ai-sdk/azure": "^1.3.23", "@eslint/js": "^9.24.0", "@himanshusinghs/google": "^1.2.11", "@jest/globals": "^30.0.0", @@ -68,6 +69,41 @@ "@himanshusinghs/ai-sdk-google": { "extraneous": true }, + "node_modules/@ai-sdk/azure": { + "version": "1.3.23", + "resolved": "https://registry.npmjs.org/@ai-sdk/azure/-/azure-1.3.23.tgz", + "integrity": "sha512-vpsaPtU24RBVk/IMM5UylR/N4RtAuL2NZLWc7LJ3tvMTHu6pI46a7w+1qIwR3F6yO9ehWR8qvfLaBefJNFxaVw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/openai": "1.3.22", + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "1.3.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-1.3.22.tgz", + "integrity": "sha512-QwA+2EkG0QyjVR+7h6FE7iOu2ivNqAVMm9UJZkVxxTk5OIq5fFJDTEI/zICEMuHImTTXR2JjsL6EirJ28Jc4cw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, "node_modules/@ai-sdk/provider": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", diff --git a/package.json b/package.json index 6eab1d11..bdcffcd0 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ }, "license": "Apache-2.0", "devDependencies": { + "@ai-sdk/azure": "^1.3.23", "@eslint/js": "^9.24.0", "@himanshusinghs/google": "^1.2.11", "@jest/globals": "^30.0.0", diff --git a/tests/accuracy/list-collections.test.ts b/tests/accuracy/list-collections.test.ts index b871a96e..2bc11dea 100644 --- a/tests/accuracy/list-collections.test.ts +++ b/tests/accuracy/list-collections.test.ts @@ -4,7 +4,7 @@ import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; function describeListCollectionsAccuracyTests(prompt: string): AccuracyTestConfig { return { - systemPrompt: "Assume that you're already connected.", + injectConnectedAssumption: true, prompt: prompt, mockedTools: { "list-collections": function listCollections() { diff --git a/tests/accuracy/list-databases.test.ts b/tests/accuracy/list-databases.test.ts index d26fbc4e..cf06303e 100644 --- a/tests/accuracy/list-databases.test.ts +++ b/tests/accuracy/list-databases.test.ts @@ -4,7 +4,7 @@ import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; function describeListDatabasesAccuracyTests(prompt: string): AccuracyTestConfig { return { - systemPrompt: "Assume that you're already connected.", + injectConnectedAssumption: true, prompt: prompt, mockedTools: { "list-databases": function listDatabases() { diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index bf99d509..28fa3bd7 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -7,6 +7,7 @@ import { appendAccuracySnapshot } from "./accuracy-snapshot.js"; export interface AccuracyTestConfig { systemPrompt?: string; + injectConnectedAssumption?: boolean; prompt: string; expectedToolCalls: ExpectedToolCall[]; mockedTools: MockedTools; @@ -44,7 +45,10 @@ export function describeAccuracyTests( eachTest("$prompt", async function (testConfig) { testTools.mockTools(testConfig.mockedTools); - const conversation = await agent.prompt(testConfig.prompt, model, testTools.vercelAiTools()); + const promptForModel = testConfig.injectConnectedAssumption + ? [testConfig.prompt, "(Assume that you are already connected to a MongoDB cluster!)"].join(" ") + : testConfig.prompt; + const conversation = await agent.prompt(promptForModel, model, testTools.vercelAiTools()); const toolCalls = testTools.getToolCalls(); const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); diff --git a/tests/accuracy/sdk/models.ts b/tests/accuracy/sdk/models.ts index e3f5ab1f..c653c79c 100644 --- a/tests/accuracy/sdk/models.ts +++ b/tests/accuracy/sdk/models.ts @@ -1,5 +1,6 @@ import { LanguageModelV1 } from "ai"; import { createGoogleGenerativeAI } from "@himanshusinghs/google"; +import { createAzure } from "@ai-sdk/azure"; import { ollama } from "ollama-ai-provider"; export interface Model

{ @@ -8,6 +9,22 @@ export interface Model

{ getModel(): P; } +export class OpenAIModel implements Model { + constructor(readonly modelName: string) {} + + isAvailable(): boolean { + return !!process.env.MDB_AZURE_OPEN_AI_API_KEY && !!process.env.MDB_AZURE_OPEN_AI_API_URL; + } + + getModel() { + return createAzure({ + baseURL: process.env.MDB_AZURE_OPEN_AI_API_URL, + apiKey: process.env.MDB_AZURE_OPEN_AI_API_KEY, + apiVersion: "2024-12-01-preview", + })(this.modelName); + } +} + export class GeminiModel implements Model { constructor(readonly modelName: string) {} @@ -35,9 +52,9 @@ export class OllamaModel implements Model { } const ALL_TESTABLE_MODELS = [ - new GeminiModel("gemini-1.5-flash"), new GeminiModel("gemini-2.0-flash"), - new OllamaModel("qwen3:1.7b"), + new OpenAIModel("gpt-4o"), + // new OllamaModel("qwen3:1.7b"), ]; export type TestableModels = ReturnType; From 9c23e950439ea3d94ab94dcea5f163a5cbaea8b9 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 14:04:21 +0200 Subject: [PATCH 08/16] chore: use ListDatabasesTool response creator for tests --- src/tools/mongodb/metadata/listDatabases.ts | 23 +++++++++++---- tests/accuracy/list-databases.test.ts | 31 ++++++++++----------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/tools/mongodb/metadata/listDatabases.ts b/src/tools/mongodb/metadata/listDatabases.ts index fe324f07..1d1ae4d2 100644 --- a/src/tools/mongodb/metadata/listDatabases.ts +++ b/src/tools/mongodb/metadata/listDatabases.ts @@ -3,6 +3,17 @@ import { MongoDBToolBase } from "../mongodbTool.js"; import * as bson from "bson"; import { OperationType } from "../../tool.js"; +export function listDatabasesResponse(databases: { name: string; sizeOnDisk: string }[]): CallToolResult { + return { + content: databases.map((db) => { + return { + text: `Name: ${db.name}, Size: ${db.sizeOnDisk} bytes`, + type: "text", + }; + }), + }; +} + export class ListDatabasesTool extends MongoDBToolBase { protected name = "list-databases"; protected description = "List all databases for a MongoDB connection"; @@ -13,13 +24,13 @@ export class ListDatabasesTool extends MongoDBToolBase { const provider = await this.ensureConnected(); const dbs = (await provider.listDatabases("")).databases as { name: string; sizeOnDisk: bson.Long }[]; - return { - content: dbs.map((db) => { + return listDatabasesResponse( + dbs.map((db) => { return { - text: `Name: ${db.name}, Size: ${db.sizeOnDisk.toString()} bytes`, - type: "text", + name: db.name, + sizeOnDisk: db.sizeOnDisk.toString(), }; - }), - }; + }) + ); } } diff --git a/tests/accuracy/list-databases.test.ts b/tests/accuracy/list-databases.test.ts index cf06303e..0a89db1d 100644 --- a/tests/accuracy/list-databases.test.ts +++ b/tests/accuracy/list-databases.test.ts @@ -1,25 +1,24 @@ import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; import { getAvailableModels } from "./sdk/models.js"; import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; +import { listDatabasesResponse } from "../../src/tools/mongodb/metadata/listDatabases.js"; -function describeListDatabasesAccuracyTests(prompt: string): AccuracyTestConfig { +function callsListDatabases(prompt: string): AccuracyTestConfig { return { injectConnectedAssumption: true, prompt: prompt, mockedTools: { "list-databases": function listDatabases() { - return { - content: [ - { - type: "text", - text: "Name: db1", - }, - { - type: "text", - text: "Name: db2", - }, - ], - }; + return listDatabasesResponse([ + { + name: "db1", + sizeOnDisk: "1024", + }, + { + name: "db2", + sizeOnDisk: "2048", + }, + ]); }, }, expectedToolCalls: [ @@ -32,7 +31,7 @@ function describeListDatabasesAccuracyTests(prompt: string): AccuracyTestConfig } describeAccuracyTests("list-databases", getAvailableModels(), [ - describeListDatabasesAccuracyTests("How many databases do I have?"), - describeListDatabasesAccuracyTests("List all the databases in my cluster."), - describeListDatabasesAccuracyTests("Is there a sample_mflix database in my cluster?"), + callsListDatabases("How many databases do I have?"), + callsListDatabases("List all the databases in my cluster."), + callsListDatabases("Is there a sample_mflix database in my cluster?"), ]); From 532c1755cffdaca41b36e117dafa7a114f889715 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 14:05:51 +0200 Subject: [PATCH 09/16] chore: use ListCollectionsTool response creators in tests --- src/tools/mongodb/metadata/listCollections.ts | 45 ++++++++------ tests/accuracy/list-collections.test.ts | 62 ++++++++++++++----- 2 files changed, 72 insertions(+), 35 deletions(-) diff --git a/src/tools/mongodb/metadata/listCollections.ts b/src/tools/mongodb/metadata/listCollections.ts index 193d0465..f676964f 100644 --- a/src/tools/mongodb/metadata/listCollections.ts +++ b/src/tools/mongodb/metadata/listCollections.ts @@ -2,6 +2,28 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { DbOperationArgs, MongoDBToolBase } from "../mongodbTool.js"; import { ToolArgs, OperationType } from "../../tool.js"; +export function listCollectionsResponse(database: string, collections: string[]): CallToolResult { + if (collections.length === 0) { + return { + content: [ + { + type: "text", + text: `No collections found for database "${database}". To create a collection, use the "create-collection" tool.`, + }, + ], + }; + } + + return { + content: collections.map((collection) => { + return { + text: `Name: "${collection}"`, + type: "text", + }; + }), + }; +} + export class ListCollectionsTool extends MongoDBToolBase { protected name = "list-collections"; protected description = "List all collections for a given database"; @@ -15,24 +37,9 @@ export class ListCollectionsTool extends MongoDBToolBase { const provider = await this.ensureConnected(); const collections = await provider.listCollections(database); - if (collections.length === 0) { - return { - content: [ - { - type: "text", - text: `No collections found for database "${database}". To create a collection, use the "create-collection" tool.`, - }, - ], - }; - } - - return { - content: collections.map((collection) => { - return { - text: `Name: "${collection.name}"`, - type: "text", - }; - }), - }; + return listCollectionsResponse( + database, + collections.map((collection) => `${collection.name}`) + ); } } diff --git a/tests/accuracy/list-collections.test.ts b/tests/accuracy/list-collections.test.ts index 2bc11dea..ac086859 100644 --- a/tests/accuracy/list-collections.test.ts +++ b/tests/accuracy/list-collections.test.ts @@ -1,25 +1,16 @@ import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; import { getAvailableModels } from "./sdk/models.js"; import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; +import { listCollectionsResponse } from "../../src/tools/mongodb/metadata/listCollections.js"; +import { listDatabasesResponse } from "../../src/tools/mongodb/metadata/listDatabases.js"; -function describeListCollectionsAccuracyTests(prompt: string): AccuracyTestConfig { +function callsListCollections(prompt: string): AccuracyTestConfig { return { injectConnectedAssumption: true, prompt: prompt, mockedTools: { "list-collections": function listCollections() { - return { - content: [ - { - type: "text", - text: "Name: coll1", - }, - { - type: "text", - text: "Name: coll1", - }, - ], - }; + return listCollectionsResponse("db1", ["coll1", "coll2"]); }, }, expectedToolCalls: [ @@ -31,8 +22,47 @@ function describeListCollectionsAccuracyTests(prompt: string): AccuracyTestConfi }; } +function callsListDatabasesAndListCollections(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "list-collections": function listCollections() { + return listCollectionsResponse("db1", ["coll1", "coll2"]); + }, + "list-databases": function listDatabases() { + return listDatabasesResponse([ + { + name: "db1", + sizeOnDisk: "1024", + }, + { + name: "db2", + sizeOnDisk: "2048", + }, + ]); + }, + }, + expectedToolCalls: [ + { + toolName: "list-databases", + parameters: {}, + }, + { + toolName: "list-collections", + parameters: { database: "db1" }, + }, + { + toolName: "list-collections", + parameters: { database: "db2" }, + }, + ], + }; +} + describeAccuracyTests("list-collections", getAvailableModels(), [ - describeListCollectionsAccuracyTests("How many collections do I have in database db1?"), - describeListCollectionsAccuracyTests("List all the collections in my MongoDB database db1."), - describeListCollectionsAccuracyTests("Is there a coll1 collection in my MongoDB database db1?"), + callsListCollections("How many collections do I have in database db1?"), + callsListCollections("List all the collections in my MongoDB database db1."), + callsListCollections("Is there a coll1 collection in my MongoDB database db1?"), + callsListDatabasesAndListCollections("List all the collections that I have in total on my cluster?"), ]); From a8f1a0c6d5f04889bddc713f88c240d22d22d787 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 14:30:50 +0200 Subject: [PATCH 10/16] chore: tests for collection-indexes tool --- src/tools/mongodb/read/collectionIndexes.ts | 74 ++++++++++++++------- tests/accuracy/collection-indexes.test.ts | 42 ++++++++++++ 2 files changed, 93 insertions(+), 23 deletions(-) create mode 100644 tests/accuracy/collection-indexes.test.ts diff --git a/src/tools/mongodb/read/collectionIndexes.ts b/src/tools/mongodb/read/collectionIndexes.ts index cc0a141b..71ade728 100644 --- a/src/tools/mongodb/read/collectionIndexes.ts +++ b/src/tools/mongodb/read/collectionIndexes.ts @@ -2,6 +2,44 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { DbOperationArgs, MongoDBToolBase } from "../mongodbTool.js"; import { ToolArgs, OperationType } from "../../tool.js"; +export function collectionIndexesResponse({ + database, + collection, + indexes = [], + namespaceNotFound, +}: { + database: string; + collection: string; + indexes?: { name: string; key: string }[]; + namespaceNotFound?: boolean; +}): CallToolResult { + if (namespaceNotFound) { + return { + content: [ + { + text: `The indexes for "${database}.${collection}" cannot be determined because the collection does not exist.`, + type: "text", + }, + ], + }; + } + + return { + content: [ + { + text: `Found ${indexes.length} indexes in the collection "${collection}":`, + type: "text", + }, + ...(indexes.map((indexDefinition) => { + return { + text: `Name "${indexDefinition.name}", definition: ${JSON.stringify(indexDefinition.key)}`, + type: "text", + }; + }) as { text: string; type: "text" }[]), + ], + }; +} + export class CollectionIndexesTool extends MongoDBToolBase { protected name = "collection-indexes"; protected description = "Describe the indexes for a collection"; @@ -11,21 +49,14 @@ export class CollectionIndexesTool extends MongoDBToolBase { protected async execute({ database, collection }: ToolArgs): Promise { const provider = await this.ensureConnected(); const indexes = await provider.getIndexes(database, collection); - - return { - content: [ - { - text: `Found ${indexes.length} indexes in the collection "${collection}":`, - type: "text", - }, - ...(indexes.map((indexDefinition) => { - return { - text: `Name "${indexDefinition.name}", definition: ${JSON.stringify(indexDefinition.key)}`, - type: "text", - }; - }) as { text: string; type: "text" }[]), - ], - }; + return collectionIndexesResponse({ + database, + collection, + indexes: indexes.map((index) => ({ + name: `${index.name}`, + key: JSON.stringify(index.key), + })), + }); } protected handleError( @@ -33,14 +64,11 @@ export class CollectionIndexesTool extends MongoDBToolBase { args: ToolArgs ): Promise | CallToolResult { if (error instanceof Error && "codeName" in error && error.codeName === "NamespaceNotFound") { - return { - content: [ - { - text: `The indexes for "${args.database}.${args.collection}" cannot be determined because the collection does not exist.`, - type: "text", - }, - ], - }; + return collectionIndexesResponse({ + database: args.database, + collection: args.collection, + namespaceNotFound: true, + }); } return super.handleError(error, args); diff --git a/tests/accuracy/collection-indexes.test.ts b/tests/accuracy/collection-indexes.test.ts new file mode 100644 index 00000000..78b223e3 --- /dev/null +++ b/tests/accuracy/collection-indexes.test.ts @@ -0,0 +1,42 @@ +import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; +import { getAvailableModels } from "./sdk/models.js"; +import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; +import { collectionIndexesResponse } from "../../src/tools/mongodb/read/collectionIndexes.js"; + +function callsCollectionIndexes(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-indexes": function collectionIndexes() { + return collectionIndexesResponse({ + database: "db1", + collection: "coll1", + indexes: [ + { + name: "year", + key: JSON.stringify({ _id: 1 }), + }, + ], + }); + }, + }, + expectedToolCalls: [ + { + toolName: "collection-indexes", + parameters: { + database: "db1", + collection: "coll1", + }, + }, + ], + }; +} + +describeAccuracyTests("collection-indexes", getAvailableModels(), [ + callsCollectionIndexes("How many indexes do I have in 'db1.coll1' namespace?"), + callsCollectionIndexes("List all the indexes in coll1 collection in db1 database"), + callsCollectionIndexes( + `Will this query: ${JSON.stringify({ year: 1994 })} on the namespace 'db1.coll1' be a collection scan?` + ), +]); From a1c0a49fe6fb5062809dd345a163ab3f2851f51d Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 16:16:20 +0200 Subject: [PATCH 11/16] modify prompt for list-collections prompt and log tools provided --- tests/accuracy/collection-indexes.test.ts | 2 +- tests/accuracy/sdk/describe-accuracy-tests.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/accuracy/collection-indexes.test.ts b/tests/accuracy/collection-indexes.test.ts index 78b223e3..683f386a 100644 --- a/tests/accuracy/collection-indexes.test.ts +++ b/tests/accuracy/collection-indexes.test.ts @@ -37,6 +37,6 @@ describeAccuracyTests("collection-indexes", getAvailableModels(), [ callsCollectionIndexes("How many indexes do I have in 'db1.coll1' namespace?"), callsCollectionIndexes("List all the indexes in coll1 collection in db1 database"), callsCollectionIndexes( - `Will this query: ${JSON.stringify({ year: 1994 })} on the namespace 'db1.coll1' be a collection scan?` + `Is the following query: ${JSON.stringify({ year: 1994 })} on the namespace 'db1.coll1' indexed?` ), ]); diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index 28fa3bd7..972f10b3 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -45,10 +45,11 @@ export function describeAccuracyTests( eachTest("$prompt", async function (testConfig) { testTools.mockTools(testConfig.mockedTools); + const toolsForModel = testTools.vercelAiTools(); const promptForModel = testConfig.injectConnectedAssumption ? [testConfig.prompt, "(Assume that you are already connected to a MongoDB cluster!)"].join(" ") : testConfig.prompt; - const conversation = await agent.prompt(promptForModel, model, testTools.vercelAiTools()); + const conversation = await agent.prompt(promptForModel, model, toolsForModel); const toolCalls = testTools.getToolCalls(); const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); @@ -73,6 +74,7 @@ export function describeAccuracyTests( expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); } catch (error) { console.warn(`Accuracy test failed for ${model.modelName} - ${suiteName} - ${testConfig.prompt}`); + console.debug(`Provided tools`, JSON.stringify(toolsForModel, null, 2)); console.debug(`Conversation`, JSON.stringify(conversation, null, 2)); console.debug(`Tool calls`, JSON.stringify(toolCalls, null, 2)); console.debug(`Tool calling accuracy`, toolCallingAccuracy); From c4169904bcfdc9806c6d7ce2317c5e17eac8303f Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 16:32:06 +0200 Subject: [PATCH 12/16] chore: have mock generators return Promise of ToolResult as well --- tests/accuracy/sdk/test-tools.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/accuracy/sdk/test-tools.ts b/tests/accuracy/sdk/test-tools.ts index 595a9069..15bb0420 100644 --- a/tests/accuracy/sdk/test-tools.ts +++ b/tests/accuracy/sdk/test-tools.ts @@ -12,7 +12,7 @@ import { Telemetry } from "../../../src/telemetry/telemetry.js"; import { Server } from "../../../src/server.js"; import { ToolCall } from "./accuracy-scorers.js"; -type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult; +type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult | Promise; export type MockedTools = Record; function getDefaultToolResultGeneratorFn(): ToolResultGeneratorFn { @@ -81,7 +81,7 @@ export class TestTools { }; } - return toolResultGeneratorFn(args); + return await toolResultGeneratorFn(args); }, }); } From 19da4f9ab719956111b6a0026250af56cea8d247 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 16:32:24 +0200 Subject: [PATCH 13/16] chore: tests for collection-schema tool --- .../mongodb/metadata/collectionSchema.ts | 59 +++++++++++-------- tests/accuracy/collection-schema.test.ts | 47 +++++++++++++++ 2 files changed, 80 insertions(+), 26 deletions(-) create mode 100644 tests/accuracy/collection-schema.test.ts diff --git a/src/tools/mongodb/metadata/collectionSchema.ts b/src/tools/mongodb/metadata/collectionSchema.ts index f0145323..71ed5256 100644 --- a/src/tools/mongodb/metadata/collectionSchema.ts +++ b/src/tools/mongodb/metadata/collectionSchema.ts @@ -1,7 +1,38 @@ import { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { DbOperationArgs, MongoDBToolBase } from "../mongodbTool.js"; import { ToolArgs, OperationType } from "../../tool.js"; -import { getSimplifiedSchema } from "mongodb-schema"; +import { getSimplifiedSchema, SimplifiedSchema } from "mongodb-schema"; + +export function collectionSchemaResponse( + database: string, + collection: string, + schema: SimplifiedSchema +): CallToolResult { + const fieldsCount = Object.entries(schema).length; + if (fieldsCount === 0) { + return { + content: [ + { + text: `Could not deduce the schema for "${database}.${collection}". This may be because it doesn't exist or is empty.`, + type: "text", + }, + ], + }; + } + + return { + content: [ + { + text: `Found ${fieldsCount} fields in the schema for "${database}.${collection}"`, + type: "text", + }, + { + text: JSON.stringify(schema), + type: "text", + }, + ], + }; +} export class CollectionSchemaTool extends MongoDBToolBase { protected name = "collection-schema"; @@ -14,30 +45,6 @@ export class CollectionSchemaTool extends MongoDBToolBase { const provider = await this.ensureConnected(); const documents = await provider.find(database, collection, {}, { limit: 5 }).toArray(); const schema = await getSimplifiedSchema(documents); - - const fieldsCount = Object.entries(schema).length; - if (fieldsCount === 0) { - return { - content: [ - { - text: `Could not deduce the schema for "${database}.${collection}". This may be because it doesn't exist or is empty.`, - type: "text", - }, - ], - }; - } - - return { - content: [ - { - text: `Found ${fieldsCount} fields in the schema for "${database}.${collection}"`, - type: "text", - }, - { - text: JSON.stringify(schema), - type: "text", - }, - ], - }; + return collectionSchemaResponse(database, collection, schema); } } diff --git a/tests/accuracy/collection-schema.test.ts b/tests/accuracy/collection-schema.test.ts new file mode 100644 index 00000000..e72c65de --- /dev/null +++ b/tests/accuracy/collection-schema.test.ts @@ -0,0 +1,47 @@ +import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; +import { getAvailableModels } from "./sdk/models.js"; +import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; +import { collectionSchemaResponse } from "../../src/tools/mongodb/metadata/collectionSchema.js"; +import { getSimplifiedSchema } from "mongodb-schema"; + +function callsCollectionSchema(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async function collectionSchema() { + return collectionSchemaResponse( + "db1", + "coll1", + await getSimplifiedSchema([ + { + name: "Sample name1", + dob: "28.11.2001", + location: "NY", + }, + { + name: "Sample name1", + dob: "28.11.2001", + location: "NY", + title: "Dr.", + }, + ]) + ); + }, + }, + expectedToolCalls: [ + { + toolName: "collection-schema", + parameters: { + database: "db1", + collection: "coll1", + }, + }, + ], + }; +} + +describeAccuracyTests("collection-schema", getAvailableModels(), [ + callsCollectionSchema("Is there a title field in 'db1.coll1' namespace?"), + callsCollectionSchema("What is the type of value stored in title field in coll1 collection in db1 database?"), +]); From c753e2c3aece18453a4f70edf9dae8fb433ec8f0 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 17:17:53 +0200 Subject: [PATCH 14/16] chore: do not fail tests on dropped accuracy --- tests/accuracy/sdk/describe-accuracy-tests.ts | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/accuracy/sdk/describe-accuracy-tests.ts b/tests/accuracy/sdk/describe-accuracy-tests.ts index 972f10b3..c602bf96 100644 --- a/tests/accuracy/sdk/describe-accuracy-tests.ts +++ b/tests/accuracy/sdk/describe-accuracy-tests.ts @@ -53,6 +53,13 @@ export function describeAccuracyTests( const toolCalls = testTools.getToolCalls(); const toolCallingAccuracy = toolCallingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); const parameterMatchingAccuracy = parameterMatchingAccuracyScorer(testConfig.expectedToolCalls, toolCalls); + console.debug(`Conversation`, JSON.stringify(conversation, null, 2)); + console.debug(`Tool calls`, JSON.stringify(toolCalls, null, 2)); + console.debug( + "Tool calling accuracy: %s, Parameter Accuracy: %s", + toolCallingAccuracy, + parameterMatchingAccuracy + ); if (accuracyDatetime && accuracyCommit) { await appendAccuracySnapshot({ datetime: accuracyDatetime, @@ -68,19 +75,6 @@ export function describeAccuracyTests( `Skipping accuracy snapshot update for ${model.modelName} - ${suiteName} - ${testConfig.prompt}` ); } - - try { - expect(toolCallingAccuracy).not.toEqual(0); - expect(parameterMatchingAccuracy).toBeGreaterThanOrEqual(0.5); - } catch (error) { - console.warn(`Accuracy test failed for ${model.modelName} - ${suiteName} - ${testConfig.prompt}`); - console.debug(`Provided tools`, JSON.stringify(toolsForModel, null, 2)); - console.debug(`Conversation`, JSON.stringify(conversation, null, 2)); - console.debug(`Tool calls`, JSON.stringify(toolCalls, null, 2)); - console.debug(`Tool calling accuracy`, toolCallingAccuracy); - console.debug(`Parameter matching accuracy`, parameterMatchingAccuracy); - throw error; - } }); }); } From 323ba955626fdf64da0b08895d6eae5c3d203391 Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Tue, 1 Jul 2025 17:17:58 +0200 Subject: [PATCH 15/16] chore: added tests for find tool --- src/tools/mongodb/read/find.ts | 36 ++++---- tests/accuracy/find.test.ts | 157 +++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 tests/accuracy/find.test.ts diff --git a/src/tools/mongodb/read/find.ts b/src/tools/mongodb/read/find.ts index 97c90e08..5e3fa4f4 100644 --- a/src/tools/mongodb/read/find.ts +++ b/src/tools/mongodb/read/find.ts @@ -13,7 +13,7 @@ export const FindArgs = { .describe("The query filter, matching the syntax of the query argument of db.collection.find()"), projection: z .record(z.string(), z.unknown()) - .optional() + // .optional() .describe("The projection, matching the syntax of the projection argument of db.collection.find()"), limit: z.number().optional().default(10).describe("The maximum number of documents to return"), sort: z @@ -22,6 +22,23 @@ export const FindArgs = { .describe("A document, describing the sort order, matching the syntax of the sort argument of cursor.sort()"), }; +export function findResponse(collection: string, documents: unknown[]): CallToolResult { + return { + content: [ + { + text: `Found ${documents.length} documents in the collection "${collection}":`, + type: "text", + }, + ...documents.map<{ type: "text"; text: string }>((doc) => { + return { + text: EJSON.stringify(doc), + type: "text", + }; + }), + ], + }; +} + export class FindTool extends MongoDBToolBase { protected name = "find"; protected description = "Run a find query against a MongoDB collection"; @@ -50,21 +67,6 @@ export class FindTool extends MongoDBToolBase { const documents = await provider.find(database, collection, filter, { projection, limit, sort }).toArray(); - const content: Array<{ text: string; type: "text" }> = [ - { - text: `Found ${documents.length} documents in the collection "${collection}":`, - type: "text", - }, - ...documents.map((doc) => { - return { - text: EJSON.stringify(doc), - type: "text", - } as { text: string; type: "text" }; - }), - ]; - - return { - content, - }; + return findResponse(collection, documents); } } diff --git a/tests/accuracy/find.test.ts b/tests/accuracy/find.test.ts new file mode 100644 index 00000000..0144e22b --- /dev/null +++ b/tests/accuracy/find.test.ts @@ -0,0 +1,157 @@ +import { describeAccuracyTests } from "./sdk/describe-accuracy-tests.js"; +import { getAvailableModels } from "./sdk/models.js"; +import { AccuracyTestConfig } from "./sdk/describe-accuracy-tests.js"; +import { findResponse } from "../../src/tools/mongodb/read/find.js"; +import { MockedTools } from "./sdk/test-tools.js"; +import { collectionSchemaResponse } from "../../src/tools/mongodb/metadata/collectionSchema.js"; +import { getSimplifiedSchema } from "mongodb-schema"; + +const documents = [ + { + title: "book1", + author: "author1", + date_of_publish: "01.01.1990", + }, + { + title: "book2", + author: "author1", + date_of_publish: "01.01.1992", + }, + { + title: "book3", + author: "author2", + date_of_publish: "01.01.1990", + }, +]; + +function callsFindNoFilter(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async () => + collectionSchemaResponse("db1", "coll1", await getSimplifiedSchema(documents)), + find: () => findResponse("coll1", documents), + }, + expectedToolCalls: [ + { + toolName: "find", + parameters: { + database: "db1", + collection: "coll1", + }, + }, + ], + }; +} + +function callsFindWithFilter(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async () => + collectionSchemaResponse("db1", "coll1", await getSimplifiedSchema(documents)), + find: () => + findResponse( + "coll1", + documents.filter((doc) => doc.author === "author1") + ), + }, + expectedToolCalls: [ + { + toolName: "find", + parameters: { + database: "db1", + collection: "coll1", + filter: { author: "author1" }, + }, + }, + ], + }; +} + +function callsFindWithProjection(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async () => + collectionSchemaResponse("db1", "coll1", await getSimplifiedSchema(documents)), + find: () => findResponse("coll1", documents), + }, + expectedToolCalls: [ + { + toolName: "find", + parameters: { + database: "db1", + collection: "coll1", + projection: { title: 1 }, + }, + }, + ], + }; +} + +function callsFindWithProjectionAndFilters(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async () => + collectionSchemaResponse("db1", "coll1", await getSimplifiedSchema(documents)), + find: () => + findResponse( + "coll1", + documents.filter((doc) => doc.date_of_publish === "01.01.1992") + ), + }, + expectedToolCalls: [ + { + toolName: "find", + parameters: { + database: "db1", + collection: "coll1", + filter: { date_of_publish: "01.01.1992" }, + projection: { title: 1 }, + }, + }, + ], + }; +} + +function callsFindWithSortAndLimit(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "collection-schema": async () => + collectionSchemaResponse("db1", "coll1", await getSimplifiedSchema(documents)), + find: () => findResponse("coll1", [documents[0], documents[1]]), + }, + expectedToolCalls: [ + { + toolName: "find", + parameters: { + database: "db1", + collection: "coll1", + sort: { date_of_publish: 1 }, + limit: 2, + }, + }, + ], + }; +} + +describeAccuracyTests("find", getAvailableModels(), [ + callsFindNoFilter("List all the documents in 'db1.coll1' namespace"), + callsFindNoFilter("Find all the documents from collection coll1 in database db1"), + callsFindWithFilter("Find all the books published by author name 'author1' in db1.coll1 namespace"), + callsFindWithFilter("Find all the documents in coll1 collection and db1 database where author is 'author1'"), + callsFindWithProjection("Give me all the title of the books available in 'db1.coll1' namespace"), + callsFindWithProjection("Give me all the title of the books published in available in 'db1.coll1' namespace"), + callsFindWithProjectionAndFilters( + "Find all the book titles from 'db1.coll1' namespace where date_of_publish is '01.01.1992'" + ), + callsFindWithSortAndLimit("List first two books sorted by the field date_of_publish in namespace db1.coll1"), +]); From eec80eb077eb919d9b8c673c89473eacfb5d8aba Mon Sep 17 00:00:00 2001 From: Himanshu Singh Date: Thu, 3 Jul 2025 14:31:46 +0200 Subject: [PATCH 16/16] chore: quick integration with braintrust sdk --- mcp-tools.json | 747 ++++++ package-lock.json | 2098 +++++++++++++++-- package.json | 5 +- .../sdk/discover-mongodb-tools-stdio.ts | 32 + tests/accuracy/sdk/discover-mongodb-tools.ts | 59 + tests/accuracy/sdk/test-tools.ts | 57 - tests/evals/eval.ts | 41 + tests/evals/tests/list-databases.eval.ts | 38 + 8 files changed, 2814 insertions(+), 263 deletions(-) create mode 100644 mcp-tools.json create mode 100644 tests/accuracy/sdk/discover-mongodb-tools-stdio.ts create mode 100644 tests/accuracy/sdk/discover-mongodb-tools.ts create mode 100644 tests/evals/eval.ts create mode 100644 tests/evals/tests/list-databases.eval.ts diff --git a/mcp-tools.json b/mcp-tools.json new file mode 100644 index 00000000..447c3ca5 --- /dev/null +++ b/mcp-tools.json @@ -0,0 +1,747 @@ +[ + { + "name": "connect", + "description": "Connect to a MongoDB instance", + "inputSchema": { + "type": "object", + "properties": { + "connectionString": { + "type": "string", + "description": "MongoDB connection string (in the mongodb:// or mongodb+srv:// format)" + } + }, + "required": [ + "connectionString" + ], + "additionalProperties": false, + "description": "Options for connecting to MongoDB.", + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "connect", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Connect to a MongoDB instance" + } + }, + { + "name": "list-collections", + "description": "List all collections for a given database", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + } + }, + "required": [ + "database" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "list-collections", + "readOnlyHint": true, + "destructiveHint": false, + "description": "List all collections for a given database" + } + }, + { + "name": "list-databases", + "description": "List all databases for a MongoDB connection", + "inputSchema": { + "type": "object", + "properties": {}, + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "list-databases", + "readOnlyHint": true, + "destructiveHint": false, + "description": "List all databases for a MongoDB connection" + } + }, + { + "name": "collection-indexes", + "description": "Describe the indexes for a collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "collection-indexes", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Describe the indexes for a collection" + } + }, + { + "name": "create-index", + "description": "Create an index for a collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "keys": { + "type": "object", + "additionalProperties": {}, + "description": "The index definition" + }, + "name": { + "type": "string", + "description": "The name of the index" + } + }, + "required": [ + "database", + "collection", + "keys" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "create-index", + "readOnlyHint": false, + "destructiveHint": false, + "description": "Create an index for a collection" + } + }, + { + "name": "collection-schema", + "description": "Describe the schema for a collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "collection-schema", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Describe the schema for a collection" + } + }, + { + "name": "find", + "description": "Run a find query against a MongoDB collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "filter": { + "type": "object", + "additionalProperties": {}, + "description": "The query filter, matching the syntax of the query argument of db.collection.find()" + }, + "projection": { + "type": "object", + "additionalProperties": {}, + "description": "The projection, matching the syntax of the projection argument of db.collection.find()" + }, + "limit": { + "type": "number", + "default": 10, + "description": "The maximum number of documents to return" + }, + "sort": { + "type": "object", + "additionalProperties": {}, + "description": "A document, describing the sort order, matching the syntax of the sort argument of cursor.sort()" + } + }, + "required": [ + "database", + "collection", + "projection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "find", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Run a find query against a MongoDB collection" + } + }, + { + "name": "insert-many", + "description": "Insert an array of documents into a MongoDB collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "documents": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": {}, + "description": "An individual MongoDB document" + }, + "description": "The array of documents to insert, matching the syntax of the document argument of db.collection.insertMany()" + } + }, + "required": [ + "database", + "collection", + "documents" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "insert-many", + "readOnlyHint": false, + "destructiveHint": false, + "description": "Insert an array of documents into a MongoDB collection" + } + }, + { + "name": "delete-many", + "description": "Removes all documents that match the filter from a MongoDB collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "filter": { + "type": "object", + "additionalProperties": {}, + "description": "The query filter, specifying the deletion criteria. Matches the syntax of the filter argument of db.collection.deleteMany()" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "delete-many", + "readOnlyHint": false, + "destructiveHint": true, + "description": "Removes all documents that match the filter from a MongoDB collection" + } + }, + { + "name": "collection-storage-size", + "description": "Gets the size of the collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "collection-storage-size", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Gets the size of the collection" + } + }, + { + "name": "count", + "description": "Gets the number of documents in a MongoDB collection using db.collection.count() and query as an optional filter parameter", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "query": { + "type": "object", + "additionalProperties": {}, + "description": "A filter/query parameter. Allows users to filter the documents to count. Matches the syntax of the filter argument of db.collection.count()." + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "count", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Gets the number of documents in a MongoDB collection using db.collection.count() and query as an optional filter parameter" + } + }, + { + "name": "db-stats", + "description": "Returns statistics that reflect the use state of a single database", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + } + }, + "required": [ + "database" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "db-stats", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Returns statistics that reflect the use state of a single database" + } + }, + { + "name": "aggregate", + "description": "Run an aggregation against a MongoDB collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "pipeline": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": {} + }, + "description": "An array of aggregation stages to execute" + } + }, + "required": [ + "database", + "collection", + "pipeline" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "aggregate", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Run an aggregation against a MongoDB collection" + } + }, + { + "name": "update-many", + "description": "Updates all documents that match the specified filter for a collection", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "filter": { + "type": "object", + "additionalProperties": {}, + "description": "The selection criteria for the update, matching the syntax of the filter argument of db.collection.updateOne()" + }, + "update": { + "type": "object", + "additionalProperties": {}, + "description": "An update document describing the modifications to apply using update operator expressions" + }, + "upsert": { + "type": "boolean", + "description": "Controls whether to insert a new document if no documents match the filter" + } + }, + "required": [ + "database", + "collection", + "update" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "update-many", + "readOnlyHint": false, + "destructiveHint": false, + "description": "Updates all documents that match the specified filter for a collection" + } + }, + { + "name": "rename-collection", + "description": "Renames a collection in a MongoDB database", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "newName": { + "type": "string", + "description": "The new name for the collection" + }, + "dropTarget": { + "type": "boolean", + "default": false, + "description": "If true, drops the target collection if it exists" + } + }, + "required": [ + "database", + "collection", + "newName" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "rename-collection", + "readOnlyHint": false, + "destructiveHint": false, + "description": "Renames a collection in a MongoDB database" + } + }, + { + "name": "drop-database", + "description": "Removes the specified database, deleting the associated data files", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + } + }, + "required": [ + "database" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "drop-database", + "readOnlyHint": false, + "destructiveHint": true, + "description": "Removes the specified database, deleting the associated data files" + } + }, + { + "name": "drop-collection", + "description": "Removes a collection or view from the database. The method also removes any indexes associated with the dropped collection.", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "drop-collection", + "readOnlyHint": false, + "destructiveHint": true, + "description": "Removes a collection or view from the database. The method also removes any indexes associated with the dropped collection." + } + }, + { + "name": "explain", + "description": "Returns statistics describing the execution of the winning plan chosen by the query optimizer for the evaluated method", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + }, + "method": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "aggregate" + }, + "arguments": { + "type": "object", + "properties": { + "pipeline": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": {} + }, + "description": "An array of aggregation stages to execute" + } + }, + "required": [ + "pipeline" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "arguments" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "find" + }, + "arguments": { + "type": "object", + "properties": { + "filter": { + "type": "object", + "additionalProperties": {}, + "description": "The query filter, matching the syntax of the query argument of db.collection.find()" + }, + "projection": { + "type": "object", + "additionalProperties": {}, + "description": "The projection, matching the syntax of the projection argument of db.collection.find()" + }, + "limit": { + "type": "number", + "default": 10, + "description": "The maximum number of documents to return" + }, + "sort": { + "type": "object", + "additionalProperties": {}, + "description": "A document, describing the sort order, matching the syntax of the sort argument of cursor.sort()" + } + }, + "required": [ + "projection" + ], + "additionalProperties": false + } + }, + "required": [ + "name", + "arguments" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "name": { + "type": "string", + "const": "count" + }, + "arguments": { + "type": "object", + "properties": { + "query": { + "type": "object", + "additionalProperties": {}, + "description": "A filter/query parameter. Allows users to filter the documents to count. Matches the syntax of the filter argument of db.collection.count()." + } + }, + "additionalProperties": false + } + }, + "required": [ + "name", + "arguments" + ], + "additionalProperties": false + } + ] + }, + "description": "The method and its arguments to run" + } + }, + "required": [ + "database", + "collection", + "method" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "explain", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Returns statistics describing the execution of the winning plan chosen by the query optimizer for the evaluated method" + } + }, + { + "name": "create-collection", + "description": "Creates a new collection in a database. If the database doesn't exist, it will be created automatically.", + "inputSchema": { + "type": "object", + "properties": { + "database": { + "type": "string", + "description": "Database name" + }, + "collection": { + "type": "string", + "description": "Collection name" + } + }, + "required": [ + "database", + "collection" + ], + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "create-collection", + "readOnlyHint": false, + "destructiveHint": false, + "description": "Creates a new collection in a database. If the database doesn't exist, it will be created automatically." + } + }, + { + "name": "mongodb-logs", + "description": "Returns the most recent logged mongod events", + "inputSchema": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "global", + "startupWarnings" + ], + "default": "global", + "description": "The type of logs to return. Global returns all recent log entries, while startupWarnings returns only warnings and errors from when the process started." + }, + "limit": { + "type": "integer", + "maximum": 1024, + "minimum": 1, + "default": 50, + "description": "The maximum number of log entries to return." + } + }, + "additionalProperties": false, + "$schema": "http://json-schema.org/draft-07/schema#" + }, + "annotations": { + "title": "mongodb-logs", + "readOnlyHint": true, + "destructiveHint": false, + "description": "Returns the most recent logged mongod events" + } + } +] \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index ecac99ab..07ec2f2c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,8 @@ "@types/simple-oauth2": "^5.0.7", "@types/yargs-parser": "^21.0.3", "ai": "^4.3.16", + "autoevals": "^0.0.130", + "braintrust": "^0.0.209", "eslint": "^9.24.0", "eslint-config-prettier": "^10.1.2", "eslint-plugin-jest": "^29.0.1", @@ -160,6 +162,195 @@ } } }, + "node_modules/@ai-sdk/solid": { + "version": "0.0.54", + "resolved": "https://registry.npmjs.org/@ai-sdk/solid/-/solid-0.0.54.tgz", + "integrity": "sha512-96KWTVK+opdFeRubqrgaJXoNiDP89gNxFRWUp0PJOotZW816AbhUf4EnDjBjXTLjXL1n0h8tGSE9sZsRkj9wQQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.50" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "solid-js": "^1.7.7" + }, + "peerDependenciesMeta": { + "solid-js": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/solid/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/solid/node_modules/@ai-sdk/provider-utils": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-1.0.22.tgz", + "integrity": "sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "eventsource-parser": "^1.1.2", + "nanoid": "^3.3.7", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/solid/node_modules/@ai-sdk/ui-utils": { + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-0.0.50.tgz", + "integrity": "sha512-Z5QYJVW+5XpSaJ4jYCCAVG7zIAuKOOdikhgpksneNmKvx61ACFaf98pmOd+xnjahl0pIlc/QIe6O4yVaJ1sEaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "json-schema": "^0.4.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/solid/node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, + "node_modules/@ai-sdk/svelte": { + "version": "0.0.57", + "resolved": "https://registry.npmjs.org/@ai-sdk/svelte/-/svelte-0.0.57.tgz", + "integrity": "sha512-SyF9ItIR9ALP9yDNAD+2/5Vl1IT6kchgyDH8xkmhysfJI6WrvJbtO1wdQ0nylvPLcsPoYu+cAlz1krU4lFHcYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.50", + "sswr": "^2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "svelte": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/svelte/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/svelte/node_modules/@ai-sdk/provider-utils": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-1.0.22.tgz", + "integrity": "sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "eventsource-parser": "^1.1.2", + "nanoid": "^3.3.7", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/svelte/node_modules/@ai-sdk/ui-utils": { + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-0.0.50.tgz", + "integrity": "sha512-Z5QYJVW+5XpSaJ4jYCCAVG7zIAuKOOdikhgpksneNmKvx61ACFaf98pmOd+xnjahl0pIlc/QIe6O4yVaJ1sEaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "json-schema": "^0.4.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/svelte/node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, "node_modules/@ai-sdk/ui-utils": { "version": "1.2.11", "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", @@ -178,6 +369,101 @@ "zod": "^3.23.8" } }, + "node_modules/@ai-sdk/vue": { + "version": "0.0.59", + "resolved": "https://registry.npmjs.org/@ai-sdk/vue/-/vue-0.0.59.tgz", + "integrity": "sha512-+ofYlnqdc8c4F6tM0IKF0+7NagZRAiqBJpGDJ+6EYhDW8FHLUP/JFBgu32SjxSxC6IKFZxEnl68ZoP/Z38EMlw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.50", + "swrv": "^1.0.4" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "vue": "^3.3.4" + }, + "peerDependenciesMeta": { + "vue": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/vue/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/vue/node_modules/@ai-sdk/provider-utils": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-1.0.22.tgz", + "integrity": "sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "eventsource-parser": "^1.1.2", + "nanoid": "^3.3.7", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/vue/node_modules/@ai-sdk/ui-utils": { + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-0.0.50.tgz", + "integrity": "sha512-Z5QYJVW+5XpSaJ4jYCCAVG7zIAuKOOdikhgpksneNmKvx61ACFaf98pmOd+xnjahl0pIlc/QIe6O4yVaJ1sEaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "json-schema": "^0.4.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/vue/node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, "node_modules/@ampproject/remapping": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", @@ -192,6 +478,19 @@ "node": ">=6.0.0" } }, + "node_modules/@asteasolutions/zod-to-openapi": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/@asteasolutions/zod-to-openapi/-/zod-to-openapi-6.4.0.tgz", + "integrity": "sha512-8cxfF7AHHx2PqnN4Cd8/O8CBu/nVYJP9DpnfVLW3BFb66VJDnqI/CczZnkqMc3SNh6J9GiX7JbJ5T4BSP4HZ2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "openapi3-ts": "^4.1.2" + }, + "peerDependencies": { + "zod": "^3.20.2" + } + }, "node_modules/@aws-crypto/sha256-browser": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", @@ -1385,6 +1684,32 @@ "dev": true, "license": "MIT" }, + "node_modules/@braintrust/core": { + "version": "0.0.89", + "resolved": "https://registry.npmjs.org/@braintrust/core/-/core-0.0.89.tgz", + "integrity": "sha512-BBLVfFxM6/d4B+i4LUTDW/FvZa4C0HN1/Cqo1W1vOflxnCJ8QVXFSqEUl+MhIU9+cJV9vwjTUVukmyscMT24hA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@asteasolutions/zod-to-openapi": "^6.3.1", + "uuid": "^9.0.1", + "zod": "^3.22.4" + } + }, + "node_modules/@braintrust/core/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1434,9 +1759,9 @@ "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.3.tgz", - "integrity": "sha512-W8bFfPA8DowP8l//sxjJLSLkD8iEjMc7cBVyP+u4cEv9sM7mdUCkgsj+t0n/BWPFtv7WWCN5Yzj0N6FJNUUqBQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.5.tgz", + "integrity": "sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA==", "cpu": [ "ppc64" ], @@ -1451,9 +1776,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.3.tgz", - "integrity": "sha512-PuwVXbnP87Tcff5I9ngV0lmiSu40xw1At6i3GsU77U7cjDDB4s0X2cyFuBiDa1SBk9DnvWwnGvVaGBqoFWPb7A==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.5.tgz", + "integrity": "sha512-AdJKSPeEHgi7/ZhuIPtcQKr5RQdo6OO2IL87JkianiMYMPbCtot9fxPbrMiBADOWWm3T2si9stAiVsGbTQFkbA==", "cpu": [ "arm" ], @@ -1468,9 +1793,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.3.tgz", - "integrity": "sha512-XelR6MzjlZuBM4f5z2IQHK6LkK34Cvv6Rj2EntER3lwCBFdg6h2lKbtRjpTTsdEjD/WSe1q8UyPBXP1x3i/wYQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.5.tgz", + "integrity": "sha512-VGzGhj4lJO+TVGV1v8ntCZWJktV7SGCs3Pn1GRWI1SBFtRALoomm8k5E9Pmwg3HOAal2VDc2F9+PM/rEY6oIDg==", "cpu": [ "arm64" ], @@ -1485,9 +1810,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.3.tgz", - "integrity": "sha512-ogtTpYHT/g1GWS/zKM0cc/tIebFjm1F9Aw1boQ2Y0eUQ+J89d0jFY//s9ei9jVIlkYi8AfOjiixcLJSGNSOAdQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.5.tgz", + "integrity": "sha512-D2GyJT1kjvO//drbRT3Hib9XPwQeWd9vZoBJn+bu/lVsOZ13cqNdDeqIF/xQ5/VmWvMduP6AmXvylO/PIc2isw==", "cpu": [ "x64" ], @@ -1502,9 +1827,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.3.tgz", - "integrity": "sha512-eESK5yfPNTqpAmDfFWNsOhmIOaQA59tAcF/EfYvo5/QWQCzXn5iUSOnqt3ra3UdzBv073ykTtmeLJZGt3HhA+w==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.5.tgz", + "integrity": "sha512-GtaBgammVvdF7aPIgH2jxMDdivezgFu6iKpmT+48+F8Hhg5J/sfnDieg0aeG/jfSvkYQU2/pceFPDKlqZzwnfQ==", "cpu": [ "arm64" ], @@ -1519,9 +1844,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.3.tgz", - "integrity": "sha512-Kd8glo7sIZtwOLcPbW0yLpKmBNWMANZhrC1r6K++uDR2zyzb6AeOYtI6udbtabmQpFaxJ8uduXMAo1gs5ozz8A==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.5.tgz", + "integrity": "sha512-1iT4FVL0dJ76/q1wd7XDsXrSW+oLoquptvh4CLR4kITDtqi2e/xwXwdCVH8hVHU43wgJdsq7Gxuzcs6Iq/7bxQ==", "cpu": [ "x64" ], @@ -1536,9 +1861,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.3.tgz", - "integrity": "sha512-EJiyS70BYybOBpJth3M0KLOus0n+RRMKTYzhYhFeMwp7e/RaajXvP+BWlmEXNk6uk+KAu46j/kaQzr6au+JcIw==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.5.tgz", + "integrity": "sha512-nk4tGP3JThz4La38Uy/gzyXtpkPW8zSAmoUhK9xKKXdBCzKODMc2adkB2+8om9BDYugz+uGV7sLmpTYzvmz6Sw==", "cpu": [ "arm64" ], @@ -1553,9 +1878,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.3.tgz", - "integrity": "sha512-Q+wSjaLpGxYf7zC0kL0nDlhsfuFkoN+EXrx2KSB33RhinWzejOd6AvgmP5JbkgXKmjhmpfgKZq24pneodYqE8Q==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.5.tgz", + "integrity": "sha512-PrikaNjiXdR2laW6OIjlbeuCPrPaAl0IwPIaRv+SMV8CiM8i2LqVUHFC1+8eORgWyY7yhQY+2U2fA55mBzReaw==", "cpu": [ "x64" ], @@ -1570,9 +1895,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.3.tgz", - "integrity": "sha512-dUOVmAUzuHy2ZOKIHIKHCm58HKzFqd+puLaS424h6I85GlSDRZIA5ycBixb3mFgM0Jdh+ZOSB6KptX30DD8YOQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.5.tgz", + "integrity": "sha512-cPzojwW2okgh7ZlRpcBEtsX7WBuqbLrNXqLU89GxWbNt6uIg78ET82qifUy3W6OVww6ZWobWub5oqZOVtwolfw==", "cpu": [ "arm" ], @@ -1587,9 +1912,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.3.tgz", - "integrity": "sha512-xCUgnNYhRD5bb1C1nqrDV1PfkwgbswTTBRbAd8aH5PhYzikdf/ddtsYyMXFfGSsb/6t6QaPSzxtbfAZr9uox4A==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.5.tgz", + "integrity": "sha512-Z9kfb1v6ZlGbWj8EJk9T6czVEjjq2ntSYLY2cw6pAZl4oKtfgQuS4HOq41M/BcoLPzrUbNd+R4BXFyH//nHxVg==", "cpu": [ "arm64" ], @@ -1604,9 +1929,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.3.tgz", - "integrity": "sha512-yplPOpczHOO4jTYKmuYuANI3WhvIPSVANGcNUeMlxH4twz/TeXuzEP41tGKNGWJjuMhotpGabeFYGAOU2ummBw==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.5.tgz", + "integrity": "sha512-sQ7l00M8bSv36GLV95BVAdhJ2QsIbCuCjh/uYrWiMQSUuV+LpXwIqhgJDcvMTj+VsQmqAHL2yYaasENvJ7CDKA==", "cpu": [ "ia32" ], @@ -1621,9 +1946,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.3.tgz", - "integrity": "sha512-P4BLP5/fjyihmXCELRGrLd793q/lBtKMQl8ARGpDxgzgIKJDRJ/u4r1A/HgpBpKpKZelGct2PGI4T+axcedf6g==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.5.tgz", + "integrity": "sha512-0ur7ae16hDUC4OL5iEnDb0tZHDxYmuQyhKhsPBV8f99f6Z9KQM02g33f93rNH5A30agMS46u2HP6qTdEt6Q1kg==", "cpu": [ "loong64" ], @@ -1638,9 +1963,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.3.tgz", - "integrity": "sha512-eRAOV2ODpu6P5divMEMa26RRqb2yUoYsuQQOuFUexUoQndm4MdpXXDBbUoKIc0iPa4aCO7gIhtnYomkn2x+bag==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.5.tgz", + "integrity": "sha512-kB/66P1OsHO5zLz0i6X0RxlQ+3cu0mkxS3TKFvkb5lin6uwZ/ttOkP3Z8lfR9mJOBk14ZwZ9182SIIWFGNmqmg==", "cpu": [ "mips64el" ], @@ -1655,9 +1980,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.3.tgz", - "integrity": "sha512-ZC4jV2p7VbzTlnl8nZKLcBkfzIf4Yad1SJM4ZMKYnJqZFD4rTI+pBG65u8ev4jk3/MPwY9DvGn50wi3uhdaghg==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.5.tgz", + "integrity": "sha512-UZCmJ7r9X2fe2D6jBmkLBMQetXPXIsZjQJCjgwpVDz+YMcS6oFR27alkgGv3Oqkv07bxdvw7fyB71/olceJhkQ==", "cpu": [ "ppc64" ], @@ -1672,9 +1997,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.3.tgz", - "integrity": "sha512-LDDODcFzNtECTrUUbVCs6j9/bDVqy7DDRsuIXJg6so+mFksgwG7ZVnTruYi5V+z3eE5y+BJZw7VvUadkbfg7QA==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.5.tgz", + "integrity": "sha512-kTxwu4mLyeOlsVIFPfQo+fQJAV9mh24xL+y+Bm6ej067sYANjyEw1dNHmvoqxJUCMnkBdKpvOn0Ahql6+4VyeA==", "cpu": [ "riscv64" ], @@ -1689,9 +2014,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.3.tgz", - "integrity": "sha512-s+w/NOY2k0yC2p9SLen+ymflgcpRkvwwa02fqmAwhBRI3SC12uiS10edHHXlVWwfAagYSY5UpmT/zISXPMW3tQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.5.tgz", + "integrity": "sha512-K2dSKTKfmdh78uJ3NcWFiqyRrimfdinS5ErLSn3vluHNeHVnBAFWC8a4X5N+7FgVE1EjXS1QDZbpqZBjfrqMTQ==", "cpu": [ "s390x" ], @@ -1706,9 +2031,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.3.tgz", - "integrity": "sha512-nQHDz4pXjSDC6UfOE1Fw9Q8d6GCAd9KdvMZpfVGWSJztYCarRgSDfOVBY5xwhQXseiyxapkiSJi/5/ja8mRFFA==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.5.tgz", + "integrity": "sha512-uhj8N2obKTE6pSZ+aMUbqq+1nXxNjZIIjCjGLfsWvVpy7gKCOL6rsY1MhRh9zLtUtAI7vpgLMK6DxjO8Qm9lJw==", "cpu": [ "x64" ], @@ -1723,9 +2048,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.3.tgz", - "integrity": "sha512-1QaLtOWq0mzK6tzzp0jRN3eccmN3hezey7mhLnzC6oNlJoUJz4nym5ZD7mDnS/LZQgkrhEbEiTn515lPeLpgWA==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.5.tgz", + "integrity": "sha512-pwHtMP9viAy1oHPvgxtOv+OkduK5ugofNTVDilIzBLpoWAM16r7b/mxBvfpuQDpRQFMfuVr5aLcn4yveGvBZvw==", "cpu": [ "arm64" ], @@ -1740,9 +2065,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.3.tgz", - "integrity": "sha512-i5Hm68HXHdgv8wkrt+10Bc50zM0/eonPb/a/OFVfB6Qvpiirco5gBA5bz7S2SHuU+Y4LWn/zehzNX14Sp4r27g==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.5.tgz", + "integrity": "sha512-WOb5fKrvVTRMfWFNCroYWWklbnXH0Q5rZppjq0vQIdlsQKuw6mdSihwSo4RV/YdQ5UCKKvBy7/0ZZYLBZKIbwQ==", "cpu": [ "x64" ], @@ -1757,9 +2082,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.3.tgz", - "integrity": "sha512-zGAVApJEYTbOC6H/3QBr2mq3upG/LBEXr85/pTtKiv2IXcgKV0RT0QA/hSXZqSvLEpXeIxah7LczB4lkiYhTAQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.5.tgz", + "integrity": "sha512-7A208+uQKgTxHd0G0uqZO8UjK2R0DDb4fDmERtARjSHWxqMTye4Erz4zZafx7Di9Cv+lNHYuncAkiGFySoD+Mw==", "cpu": [ "arm64" ], @@ -1774,9 +2099,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.3.tgz", - "integrity": "sha512-fpqctI45NnCIDKBH5AXQBsD0NDPbEFczK98hk/aa6HJxbl+UtLkJV2+Bvy5hLSLk3LHmqt0NTkKNso1A9y1a4w==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.5.tgz", + "integrity": "sha512-G4hE405ErTWraiZ8UiSoesH8DaCsMm0Cay4fsFWOOUcz8b8rC6uCvnagr+gnioEjWn0wC+o1/TAHt+It+MpIMg==", "cpu": [ "x64" ], @@ -1791,9 +2116,9 @@ } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.3.tgz", - "integrity": "sha512-ROJhm7d8bk9dMCUZjkS8fgzsPAZEjtRJqCAmVgB0gMrvG7hfmPmz9k1rwO4jSiblFjYmNvbECL9uhaPzONMfgA==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.5.tgz", + "integrity": "sha512-l+azKShMy7FxzY0Rj4RCt5VD/q8mG/e+mDivgspo+yL8zW7qEwctQ6YqKX34DTEleFAvCIUviCFX1SDZRSyMQA==", "cpu": [ "x64" ], @@ -1808,9 +2133,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.3.tgz", - "integrity": "sha512-YWcow8peiHpNBiIXHwaswPnAXLsLVygFwCB3A7Bh5jRkIBFWHGmNQ48AlX4xDvQNoMZlPYzjVOQDYEzWCqufMQ==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.5.tgz", + "integrity": "sha512-O2S7SNZzdcFG7eFKgvwUEZ2VG9D/sn/eIiz8XRZ1Q/DO5a3s76Xv0mdBzVM5j5R639lXQmPmSo0iRpHqUUrsxw==", "cpu": [ "arm64" ], @@ -1825,9 +2150,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.3.tgz", - "integrity": "sha512-qspTZOIGoXVS4DpNqUYUs9UxVb04khS1Degaw/MnfMe7goQ3lTfQ13Vw4qY/Nj0979BGvMRpAYbs/BAxEvU8ew==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.5.tgz", + "integrity": "sha512-onOJ02pqs9h1iMJ1PQphR+VZv8qBMQ77Klcsqv9CNW2w6yLqoURLcgERAIurY6QE63bbLuqgP9ATqajFLK5AMQ==", "cpu": [ "ia32" ], @@ -1842,9 +2167,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.3.tgz", - "integrity": "sha512-ICgUR+kPimx0vvRzf+N/7L7tVSQeE3BYY+NhHRHXS1kBuPO7z2+7ea2HbhDyZdTephgvNvKrlDDKUexuCVBVvg==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.5.tgz", + "integrity": "sha512-TXv6YnJ8ZMVdX+SXWVBo/0p8LTcrUYngpWjvm91TMjjBQii7Oz11Lw5lbDV5Y0TzuhSJHwiH4hEtC1I42mMS0g==", "cpu": [ "x64" ], @@ -3365,6 +3690,23 @@ "jsep": "^0.4.0||^1.0.0" } }, + "node_modules/@kwsites/file-exists": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/file-exists/-/file-exists-1.1.1.tgz", + "integrity": "sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.1" + } + }, + "node_modules/@kwsites/promise-deferred": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz", + "integrity": "sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==", + "dev": true, + "license": "MIT" + }, "node_modules/@modelcontextprotocol/inspector": { "version": "0.14.1", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/inspector/-/inspector-0.14.1.tgz", @@ -4206,6 +4548,13 @@ "node": ">=14.15.1" } }, + "node_modules/@next/env": { + "version": "14.2.30", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.30.tgz", + "integrity": "sha512-KBiBKrDY6kxTQWGzKjQB7QirL3PiiOkV7KW98leHFjtVRKtft76Ra5qSA/SL75xT44dp6hOcqiiJ6iievLOYug==", + "dev": true, + "license": "MIT" + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -6300,6 +6649,17 @@ "node": ">=18.0.0" } }, + "node_modules/@sveltejs/acorn-typescript": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@sveltejs/acorn-typescript/-/acorn-typescript-1.0.5.tgz", + "integrity": "sha512-IwQk4yfwLdibDlrXVE04jTZYlLnwsTT2PIOQQGNLWfjavGifnk1JD1LcZjZaBTRcxZu2FfPfNLOE04DSu9lqtQ==", + "dev": true, + "license": "MIT", + "peer": true, + "peerDependencies": { + "acorn": "^8.9.0" + } + }, "node_modules/@tootallnate/quickjs-emscripten": { "version": "0.23.0", "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", @@ -6458,6 +6818,17 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/simple-oauth2": { "version": "5.0.7", "resolved": "https://registry.npmjs.org/@types/simple-oauth2/-/simple-oauth2-5.0.7.tgz", @@ -6773,19 +7144,185 @@ "dev": true, "license": "ISC" }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "node_modules/@vercel/functions": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@vercel/functions/-/functions-1.6.0.tgz", + "integrity": "sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==", "dev": true, - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, + "license": "Apache-2.0", "engines": { - "node": ">=6.5" - } - }, + "node": ">= 16" + }, + "peerDependencies": { + "@aws-sdk/credential-provider-web-identity": "*" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-provider-web-identity": { + "optional": true + } + } + }, + "node_modules/@vue/compiler-core": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.5.17.tgz", + "integrity": "sha512-Xe+AittLbAyV0pabcN7cP7/BenRBNcteM4aSDCtRvGw0d9OL+HG1u/XHLY/kt1q4fyMeZYXyIYrsHuPSiDPosA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/parser": "^7.27.5", + "@vue/shared": "3.5.17", + "entities": "^4.5.0", + "estree-walker": "^2.0.2", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-dom": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.5.17.tgz", + "integrity": "sha512-+2UgfLKoaNLhgfhV5Ihnk6wB4ljyW1/7wUIog2puUqajiC29Lp5R/IKDdkebh9jTbTogTbsgB+OY9cEWzG95JQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/compiler-core": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/compiler-sfc": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.5.17.tgz", + "integrity": "sha512-rQQxbRJMgTqwRugtjw0cnyQv9cP4/4BxWfTdRBkqsTfLOHWykLzbOc3C4GGzAmdMDxhzU/1Ija5bTjMVrddqww==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@babel/parser": "^7.27.5", + "@vue/compiler-core": "3.5.17", + "@vue/compiler-dom": "3.5.17", + "@vue/compiler-ssr": "3.5.17", + "@vue/shared": "3.5.17", + "estree-walker": "^2.0.2", + "magic-string": "^0.30.17", + "postcss": "^8.5.6", + "source-map-js": "^1.2.1" + } + }, + "node_modules/@vue/compiler-sfc/node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "peer": true, + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/@vue/compiler-ssr": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.5.17.tgz", + "integrity": "sha512-hkDbA0Q20ZzGgpj5uZjb9rBzQtIHLS78mMilwrlpWk2Ep37DYntUz0PonQ6kr113vfOEdM+zTBuJDaceNIW0tQ==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/compiler-dom": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/reactivity": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/reactivity/-/reactivity-3.5.17.tgz", + "integrity": "sha512-l/rmw2STIscWi7SNJp708FK4Kofs97zc/5aEPQh4bOsReD/8ICuBcEmS7KGwDj5ODQLYWVN2lNibKJL1z5b+Lw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/runtime-core": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/runtime-core/-/runtime-core-3.5.17.tgz", + "integrity": "sha512-QQLXa20dHg1R0ri4bjKeGFKEkJA7MMBxrKo2G+gJikmumRS7PTD4BOU9FKrDQWMKowz7frJJGqBffYMgQYS96Q==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/reactivity": "3.5.17", + "@vue/shared": "3.5.17" + } + }, + "node_modules/@vue/runtime-dom": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/runtime-dom/-/runtime-dom-3.5.17.tgz", + "integrity": "sha512-8El0M60TcwZ1QMz4/os2MdlQECgGoVHPuLnQBU3m9h3gdNRW9xRmI8iLS4t/22OQlOE6aJvNNlBiCzPHur4H9g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/reactivity": "3.5.17", + "@vue/runtime-core": "3.5.17", + "@vue/shared": "3.5.17", + "csstype": "^3.1.3" + } + }, + "node_modules/@vue/server-renderer": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/server-renderer/-/server-renderer-3.5.17.tgz", + "integrity": "sha512-BOHhm8HalujY6lmC3DbqF6uXN/K00uWiEeF22LfEsm9Q93XeJ/plHTepGwf6tqFcF7GA5oGSSAAUock3VvzaCA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/compiler-ssr": "3.5.17", + "@vue/shared": "3.5.17" + }, + "peerDependencies": { + "vue": "3.5.17" + } + }, + "node_modules/@vue/shared": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.5.17.tgz", + "integrity": "sha512-CabR+UN630VnsJO/jHWYBC1YVXyMq94KKp6iF5MQgZJs5I8cmjw6oVMO1oDbtBkENSHSSn/UadWlW/OAgdmKrg==", + "dev": true, + "license": "MIT", + "peer": true + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -6844,6 +7381,19 @@ "node": ">= 14" } }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ai": { "version": "4.3.16", "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.16.tgz", @@ -6877,7 +7427,6 @@ "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -6982,6 +7531,17 @@ "node": ">=10" } }, + "node_modules/aria-query": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", + "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -7023,12 +7583,41 @@ "dev": true, "license": "MIT" }, + "node_modules/autoevals": { + "version": "0.0.130", + "resolved": "https://registry.npmjs.org/autoevals/-/autoevals-0.0.130.tgz", + "integrity": "sha512-JS0T/YCEH13AAOGiWWGJDkIPP8LsDmRBYr3EazTukHxvd0nidOW7fGj0qVPFx2bARrSNO9AfCR6xoTP/5m3Bmw==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.13.0", + "compute-cosine-similarity": "^1.1.0", + "js-levenshtein": "^1.1.6", + "js-yaml": "^4.1.0", + "linear-sum-assignment": "^1.0.7", + "mustache": "^4.2.0", + "openai": "^4.47.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.5" + } + }, "node_modules/aws4": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.2.tgz", "integrity": "sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw==", "license": "MIT" }, + "node_modules/axobject-query": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", + "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", + "dev": true, + "license": "Apache-2.0", + "peer": true, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -7215,147 +7804,750 @@ "leven": "^3.1.0 < 4" }, "engines": { - "node": ">= 12.13.0" - }, - "peerDependencies": { - "ajv": "4.11.8 - 8" + "node": ">= 12.13.0" + }, + "peerDependencies": { + "ajv": "4.11.8 - 8" + } + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/binary-search": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", + "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", + "dev": true, + "license": "CC0-1.0" + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/bl/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/bl/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/body-parser": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", + "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.0", + "http-errors": "^2.0.0", + "iconv-lite": "^0.6.3", + "on-finished": "^2.4.1", + "qs": "^6.14.0", + "raw-body": "^3.0.0", + "type-is": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", + "license": "MIT" + }, + "node_modules/bplist-parser": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", + "integrity": "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.44" + }, + "engines": { + "node": ">= 5.10.0" + } + }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/braintrust": { + "version": "0.0.209", + "resolved": "https://registry.npmjs.org/braintrust/-/braintrust-0.0.209.tgz", + "integrity": "sha512-acsjb06ttD/gllfb59idiq1lDAdvsoHcHSJPkmddSIRPORy3vIYt3kfKXiW+WlhwZs2pl3lN8X8pTVuLyj5NNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ai-sdk/provider": "^1.0.1", + "@braintrust/core": "0.0.89", + "@next/env": "^14.2.3", + "@vercel/functions": "^1.0.2", + "ai": "^3.2.16", + "argparse": "^2.0.1", + "chalk": "^4.1.2", + "cli-progress": "^3.12.0", + "cors": "^2.8.5", + "dotenv": "^16.4.5", + "esbuild": "^0.25.5", + "eventsource-parser": "^1.1.2", + "express": "^4.21.2", + "graceful-fs": "^4.2.11", + "http-errors": "^2.0.0", + "minimatch": "^9.0.3", + "mustache": "^4.2.0", + "pluralize": "^8.0.0", + "simple-git": "^3.21.0", + "slugify": "^1.6.6", + "source-map": "^0.7.4", + "uuid": "^9.0.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.5" + }, + "bin": { + "braintrust": "dist/cli.js" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/braintrust/node_modules/@ai-sdk/provider-utils": { + "version": "1.0.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-1.0.22.tgz", + "integrity": "sha512-YHK2rpj++wnLVc9vPGzGFP3Pjeld2MwhKinetA0zKXOoHAT/Jit5O8kZsxcSlJPu9wvcGT1UGZEjZrtO7PfFOQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "eventsource-parser": "^1.1.2", + "nanoid": "^3.3.7", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/braintrust/node_modules/@ai-sdk/provider-utils/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/braintrust/node_modules/@ai-sdk/react": { + "version": "0.0.70", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-0.0.70.tgz", + "integrity": "sha512-GnwbtjW4/4z7MleLiW+TOZC2M29eCg1tOUpuEiYFMmFNZK8mkrqM0PFZMo6UsYeUYMWqEOOcPOU9OQVJMJh7IQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/ui-utils": "0.0.50", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/braintrust/node_modules/@ai-sdk/ui-utils": { + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-0.0.50.tgz", + "integrity": "sha512-Z5QYJVW+5XpSaJ4jYCCAVG7zIAuKOOdikhgpksneNmKvx61ACFaf98pmOd+xnjahl0pIlc/QIe6O4yVaJ1sEaw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "json-schema": "^0.4.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/braintrust/node_modules/@ai-sdk/ui-utils/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/braintrust/node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/ai": { + "version": "3.4.33", + "resolved": "https://registry.npmjs.org/ai/-/ai-3.4.33.tgz", + "integrity": "sha512-plBlrVZKwPoRTmM8+D1sJac9Bq8eaa2jiZlHLZIWekKWI1yMWYZvCCEezY9ASPwRhULYDJB2VhKOBUUeg3S5JQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "0.0.26", + "@ai-sdk/provider-utils": "1.0.22", + "@ai-sdk/react": "0.0.70", + "@ai-sdk/solid": "0.0.54", + "@ai-sdk/svelte": "0.0.57", + "@ai-sdk/ui-utils": "0.0.50", + "@ai-sdk/vue": "0.0.59", + "@opentelemetry/api": "1.9.0", + "eventsource-parser": "1.1.2", + "json-schema": "^0.4.0", + "jsondiffpatch": "0.6.0", + "secure-json-parse": "^2.7.0", + "zod-to-json-schema": "^3.23.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "openai": "^4.42.0", + "react": "^18 || ^19 || ^19.0.0-rc", + "sswr": "^2.1.0", + "svelte": "^3.0.0 || ^4.0.0 || ^5.0.0", + "zod": "^3.0.0" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + }, + "react": { + "optional": true + }, + "sswr": { + "optional": true + }, + "svelte": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/braintrust/node_modules/ai/node_modules/@ai-sdk/provider": { + "version": "0.0.26", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-0.0.26.tgz", + "integrity": "sha512-dQkfBDs2lTYpKM8389oopPdQgIU007GQyCbuPPrV+K6MtSII3HBfE0stUIMXUb44L+LK1t6GXPP7wjSzjO6uKg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/braintrust/node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/braintrust/node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/braintrust/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/braintrust/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true, + "license": "MIT" + }, + "node_modules/braintrust/node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.18" + } + }, + "node_modules/braintrust/node_modules/express": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/braintrust/node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/braintrust/node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" } }, - "node_modules/big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", - "license": "Unlicense", + "node_modules/braintrust/node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "license": "MIT", "engines": { - "node": ">=0.6" + "node": ">= 0.6" } }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "node_modules/braintrust/node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", "dev": true, "license": "MIT", - "engines": { - "node": ">=8" - }, "funding": { "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "node_modules/braintrust/node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, "license": "MIT", - "optional": true, - "dependencies": { - "file-uri-to-path": "1.0.0" + "engines": { + "node": ">= 0.6" } }, - "node_modules/bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "node_modules/braintrust/node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", "dev": true, "license": "MIT", "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" } }, - "node_modules/bl/node_modules/readable-stream": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", - "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "node_modules/braintrust/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, - "license": "MIT", + "license": "ISC", "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/bl/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "node_modules/braintrust/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", "dev": true, "license": "MIT" }, - "node_modules/bl/node_modules/string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "node_modules/braintrust/node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/braintrust/node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, "license": "MIT", "dependencies": { - "safe-buffer": "~5.1.0" + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" } }, - "node_modules/body-parser": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.0.tgz", - "integrity": "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==", + "node_modules/braintrust/node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dev": true, "license": "MIT", "dependencies": { - "bytes": "^3.1.2", - "content-type": "^1.0.5", - "debug": "^4.4.0", - "http-errors": "^2.0.0", - "iconv-lite": "^0.6.3", - "on-finished": "^2.4.1", - "qs": "^6.14.0", - "raw-body": "^3.0.0", - "type-is": "^2.0.0" + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" }, "engines": { - "node": ">=18" + "node": ">= 0.8.0" } }, - "node_modules/bowser": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", - "license": "MIT" + "node_modules/braintrust/node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } }, - "node_modules/bplist-parser": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", - "integrity": "sha512-z0M+byMThzQmD9NILRniCUXYsYpjwnlO8N5uCFaCqIOpqRsJCrQL9NK3JsD67CN5a08nF5oIL2bD6loTdHOuKw==", + "node_modules/braintrust/node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dev": true, "license": "MIT", "dependencies": { - "big-integer": "^1.6.44" + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" }, "engines": { - "node": ">= 5.10.0" + "node": ">= 0.8.0" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "node_modules/braintrust/node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", "dev": true, "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" + "engines": { + "node": ">=8.0.0" } }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "node_modules/braintrust/node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 8" + } + }, + "node_modules/braintrust/node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", "dev": true, "license": "MIT", "dependencies": { - "fill-range": "^7.1.1" + "media-typer": "0.3.0", + "mime-types": "~2.1.24" }, "engines": { - "node": ">=8" + "node": ">= 0.6" + } + }, + "node_modules/braintrust/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" } }, "node_modules/browserslist": { @@ -7644,6 +8836,13 @@ "node": ">=10" } }, + "node_modules/cheminfo-types": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/cheminfo-types/-/cheminfo-types-1.8.1.tgz", + "integrity": "sha512-FRcpVkox+cRovffgqNdDFQ1eUav+i/Vq/CUd1hcfEl2bevntFlzznL+jE8g4twl6ElB7gZjCko6pYpXyMn+6dA==", + "dev": true, + "license": "MIT" + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -7722,6 +8921,19 @@ "dev": true, "license": "MIT" }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/cli-table": { "version": "0.3.11", "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.11.tgz", @@ -7851,6 +9063,38 @@ "node": ">=18" } }, + "node_modules/compute-cosine-similarity": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-cosine-similarity/-/compute-cosine-similarity-1.1.0.tgz", + "integrity": "sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==", + "dev": true, + "dependencies": { + "compute-dot": "^1.1.0", + "compute-l2norm": "^1.1.0", + "validate.io-array": "^1.0.5", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-dot": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-dot/-/compute-dot-1.1.0.tgz", + "integrity": "sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==", + "dev": true, + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, + "node_modules/compute-l2norm": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/compute-l2norm/-/compute-l2norm-1.1.0.tgz", + "integrity": "sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==", + "dev": true, + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -8752,6 +9996,20 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -8816,9 +10074,9 @@ "license": "MIT" }, "node_modules/esbuild": { - "version": "0.25.3", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.3.tgz", - "integrity": "sha512-qKA6Pvai73+M2FtftpNKRxJ78GIjmFXFxd/1DVBqGo/qNhLSfv+G12n9pNoWdytJC8U00TrViOwpjT0zgqQS8Q==", + "version": "0.25.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.5.tgz", + "integrity": "sha512-P8OtKZRv/5J5hhz0cUAdu/cLuPIKXpQl1R9pZtvmHWQvrAUVd0UNIPT4IB4W3rNOqVO0rlqHmCIbSwxh/c9yUQ==", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -8829,31 +10087,31 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.25.3", - "@esbuild/android-arm": "0.25.3", - "@esbuild/android-arm64": "0.25.3", - "@esbuild/android-x64": "0.25.3", - "@esbuild/darwin-arm64": "0.25.3", - "@esbuild/darwin-x64": "0.25.3", - "@esbuild/freebsd-arm64": "0.25.3", - "@esbuild/freebsd-x64": "0.25.3", - "@esbuild/linux-arm": "0.25.3", - "@esbuild/linux-arm64": "0.25.3", - "@esbuild/linux-ia32": "0.25.3", - "@esbuild/linux-loong64": "0.25.3", - "@esbuild/linux-mips64el": "0.25.3", - "@esbuild/linux-ppc64": "0.25.3", - "@esbuild/linux-riscv64": "0.25.3", - "@esbuild/linux-s390x": "0.25.3", - "@esbuild/linux-x64": "0.25.3", - "@esbuild/netbsd-arm64": "0.25.3", - "@esbuild/netbsd-x64": "0.25.3", - "@esbuild/openbsd-arm64": "0.25.3", - "@esbuild/openbsd-x64": "0.25.3", - "@esbuild/sunos-x64": "0.25.3", - "@esbuild/win32-arm64": "0.25.3", - "@esbuild/win32-ia32": "0.25.3", - "@esbuild/win32-x64": "0.25.3" + "@esbuild/aix-ppc64": "0.25.5", + "@esbuild/android-arm": "0.25.5", + "@esbuild/android-arm64": "0.25.5", + "@esbuild/android-x64": "0.25.5", + "@esbuild/darwin-arm64": "0.25.5", + "@esbuild/darwin-x64": "0.25.5", + "@esbuild/freebsd-arm64": "0.25.5", + "@esbuild/freebsd-x64": "0.25.5", + "@esbuild/linux-arm": "0.25.5", + "@esbuild/linux-arm64": "0.25.5", + "@esbuild/linux-ia32": "0.25.5", + "@esbuild/linux-loong64": "0.25.5", + "@esbuild/linux-mips64el": "0.25.5", + "@esbuild/linux-ppc64": "0.25.5", + "@esbuild/linux-riscv64": "0.25.5", + "@esbuild/linux-s390x": "0.25.5", + "@esbuild/linux-x64": "0.25.5", + "@esbuild/netbsd-arm64": "0.25.5", + "@esbuild/netbsd-x64": "0.25.5", + "@esbuild/openbsd-arm64": "0.25.5", + "@esbuild/openbsd-x64": "0.25.5", + "@esbuild/sunos-x64": "0.25.5", + "@esbuild/win32-arm64": "0.25.5", + "@esbuild/win32-ia32": "0.25.5", + "@esbuild/win32-x64": "0.25.5" } }, "node_modules/escalade": { @@ -9131,6 +10389,14 @@ "node": "*" } }, + "node_modules/esm-env": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.2.tgz", + "integrity": "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/espree": { "version": "10.4.0", "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", @@ -9175,6 +10441,17 @@ "node": ">=0.10" } }, + "node_modules/esrap": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/esrap/-/esrap-2.0.1.tgz", + "integrity": "sha512-6n1JodkxeMvyTDCog7J//t8Yti//fGicZgtFLko6h/aEpc54BK9O8k9cZgC2J8+2Dh1U5uYIxuJWSsylybvFBA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.4.15" + } + }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -9197,6 +10474,14 @@ "node": ">=4.0" } }, + "node_modules/estree-walker": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", + "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", @@ -9434,8 +10719,7 @@ "url": "https://opencollective.com/fastify" } ], - "license": "BSD-3-Clause", - "peer": true + "license": "BSD-3-Clause" }, "node_modules/fast-xml-parser": { "version": "4.4.1", @@ -9512,6 +10796,13 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fft.js": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/fft.js/-/fft.js-4.0.4.tgz", + "integrity": "sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==", + "dev": true, + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -9643,6 +10934,13 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "dev": true, + "license": "MIT" + }, "node_modules/form-data/node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -9666,6 +10964,30 @@ "node": ">= 0.6" } }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -10143,6 +11465,16 @@ "node": ">=10.17.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -10271,6 +11603,16 @@ "license": "ISC", "optional": true }, + "node_modules/install": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/install/-/install-0.13.0.tgz", + "integrity": "sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/ip-address": { "version": "9.0.5", "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", @@ -10300,6 +11642,13 @@ "license": "MIT", "optional": true }, + "node_modules/is-any-array": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", + "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", + "dev": true, + "license": "MIT" + }, "node_modules/is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -10435,6 +11784,17 @@ "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", "license": "MIT" }, + "node_modules/is-reference": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-3.0.3.tgz", + "integrity": "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@types/estree": "^1.0.6" + } + }, "node_modules/is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", @@ -11540,6 +12900,19 @@ "node": ">= 0.8.0" } }, + "node_modules/linear-sum-assignment": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/linear-sum-assignment/-/linear-sum-assignment-1.0.7.tgz", + "integrity": "sha512-jfLoSGwZNyjfY8eK4ayhjfcIu3BfWvP6sWieYzYI3AWldwXVoWEz1gtrQL10v/8YltYLBunqNjeVFXPMUs+MJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cheminfo-types": "^1.7.3", + "install": "^0.13.0", + "ml-matrix": "^6.11.0", + "ml-spectra-processing": "^14.2.2" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", @@ -11547,6 +12920,14 @@ "dev": true, "license": "MIT" }, + "node_modules/locate-character": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-character/-/locate-character-3.0.0.tgz", + "integrity": "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -11651,6 +13032,17 @@ "license": "MIT", "optional": true }, + "node_modules/magic-string": { + "version": "0.30.17", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.17.tgz", + "integrity": "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0" + } + }, "node_modules/make-dir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", @@ -11914,6 +13306,71 @@ "license": "MIT", "optional": true }, + "node_modules/ml-array-max": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/ml-array-max/-/ml-array-max-1.2.4.tgz", + "integrity": "sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-min": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/ml-array-min/-/ml-array-min-1.2.3.tgz", + "integrity": "sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0" + } + }, + "node_modules/ml-array-rescale": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/ml-array-rescale/-/ml-array-rescale-1.3.7.tgz", + "integrity": "sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.0", + "ml-array-max": "^1.2.4", + "ml-array-min": "^1.2.3" + } + }, + "node_modules/ml-matrix": { + "version": "6.12.1", + "resolved": "https://registry.npmjs.org/ml-matrix/-/ml-matrix-6.12.1.tgz", + "integrity": "sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-any-array": "^2.0.1", + "ml-array-rescale": "^1.3.7" + } + }, + "node_modules/ml-spectra-processing": { + "version": "14.12.0", + "resolved": "https://registry.npmjs.org/ml-spectra-processing/-/ml-spectra-processing-14.12.0.tgz", + "integrity": "sha512-RoJj2r4tGElyPDwBzmoCa+j3rLomBzz+JHGVPxf1tASAE82NkjgvuCFZFay+g0DXTkxDGYFxor+zayqA4nQrng==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-search": "^1.3.6", + "cheminfo-types": "^1.8.1", + "fft.js": "^4.0.4", + "is-any-array": "^2.0.1", + "ml-matrix": "^6.12.1", + "ml-xsadd": "^3.0.1" + } + }, + "node_modules/ml-xsadd": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ml-xsadd/-/ml-xsadd-3.0.1.tgz", + "integrity": "sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==", + "dev": true, + "license": "MIT" + }, "node_modules/mobx": { "version": "6.13.7", "resolved": "https://registry.npmjs.org/mobx/-/mobx-6.13.7.tgz", @@ -12286,6 +13743,16 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "license": "MIT" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nan": { "version": "2.22.2", "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.2.tgz", @@ -12705,6 +14172,100 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openai": { + "version": "4.104.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.104.0.tgz", + "integrity": "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/@types/node": { + "version": "18.19.115", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.115.tgz", + "integrity": "sha512-kNrFiTgG4a9JAn1LMQeLOv3MvXIPokzXziohMrMsvpYgLpdEt/mMiVYc4sGKtDfyxM5gIDF4VgrPRyCw4fHOYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/openai/node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/openai/node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dev": true, + "license": "MIT" + }, + "node_modules/openai/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/openai/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true, + "license": "BSD-2-Clause" + }, + "node_modules/openai/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/openapi-fetch": { "version": "0.14.0", "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.14.0.tgz", @@ -12833,6 +14394,16 @@ "node": ">=12" } }, + "node_modules/openapi3-ts": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/openapi3-ts/-/openapi3-ts-4.5.0.tgz", + "integrity": "sha512-jaL+HgTq2Gj5jRcfdutgRGLosCy/hT8sQf6VOy+P+g36cZOjI1iukdPnijC+4CmeRzg/jEllJUboEic2FhxhtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.8.0" + } + }, "node_modules/openid-client": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/openid-client/-/openid-client-5.7.1.tgz", @@ -14479,6 +16050,22 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-git": { + "version": "3.28.0", + "resolved": "https://registry.npmjs.org/simple-git/-/simple-git-3.28.0.tgz", + "integrity": "sha512-Rs/vQRwsn1ILH1oBUy8NucJlXmnnLeLCfcvbSehkPzbv3wwoFWIdtfd6Ndo6ZPhlPsCZ60CPI4rxurnwAa+a2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@kwsites/file-exists": "^1.1.1", + "@kwsites/promise-deferred": "^1.1.1", + "debug": "^4.4.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/git-js?sponsor=1" + } + }, "node_modules/simple-oauth2": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/simple-oauth2/-/simple-oauth2-5.1.0.tgz", @@ -14680,6 +16267,19 @@ "nan": "^2.20.0" } }, + "node_modules/sswr": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/sswr/-/sswr-2.2.0.tgz", + "integrity": "sha512-clTszLPZkmycALTHD1mXGU+mOtA/MIoLgS1KGTTzFNVm9rytQVykgRaP+z1zl572cz0bTqj4rFVoC2N+IGK4Sg==", + "dev": true, + "license": "MIT", + "dependencies": { + "swrev": "^4.0.0" + }, + "peerDependencies": { + "svelte": "^4.0.0 || ^5.0.0" + } + }, "node_modules/stack-utils": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", @@ -14906,6 +16506,33 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/svelte": { + "version": "5.35.1", + "resolved": "https://registry.npmjs.org/svelte/-/svelte-5.35.1.tgz", + "integrity": "sha512-3kNMwstMB2VUBod63H6BQPzBr7NiPRR9qFVzrWqW/nU4ulqqAYZsJ6E7m8LYFoRzuW6yylx+uzdgKVhFKZVf4Q==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@ampproject/remapping": "^2.3.0", + "@jridgewell/sourcemap-codec": "^1.5.0", + "@sveltejs/acorn-typescript": "^1.0.5", + "@types/estree": "^1.0.5", + "acorn": "^8.12.1", + "aria-query": "^5.3.1", + "axobject-query": "^4.1.0", + "clsx": "^2.1.1", + "esm-env": "^1.2.1", + "esrap": "^2.0.0", + "is-reference": "^3.0.3", + "locate-character": "^3.0.0", + "magic-string": "^0.30.11", + "zimmerframe": "^1.1.2" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/swagger2openapi": { "version": "7.0.8", "resolved": "https://registry.npmjs.org/swagger2openapi/-/swagger2openapi-7.0.8.tgz", @@ -15004,6 +16631,23 @@ "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/swrev": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/swrev/-/swrev-4.0.0.tgz", + "integrity": "sha512-LqVcOHSB4cPGgitD1riJ1Hh4vdmITOp+BkmfmXRh4hSF/t7EnS4iD+SOTmq7w5pPm/SiPeto4ADbKS6dHUDWFA==", + "dev": true, + "license": "MIT" + }, + "node_modules/swrv": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/swrv/-/swrv-1.1.0.tgz", + "integrity": "sha512-pjllRDr2s0iTwiE5Isvip51dZGR7GjLH1gCSVyE8bQnbAx6xackXsFdojau+1O5u98yHF5V73HQGOFxKUXO9gQ==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "vue": ">=3.2.26 < 4" + } + }, "node_modules/synckit": { "version": "0.11.8", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.11.8.tgz", @@ -15800,6 +17444,19 @@ "node": ">=10.12.0" } }, + "node_modules/validate.io-array": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", + "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==", + "dev": true, + "license": "MIT" + }, + "node_modules/validate.io-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", + "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==", + "dev": true + }, "node_modules/vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", @@ -15809,6 +17466,29 @@ "node": ">= 0.8" } }, + "node_modules/vue": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/vue/-/vue-3.5.17.tgz", + "integrity": "sha512-LbHV3xPN9BeljML+Xctq4lbz2lVHCR6DtbpTf5XIO6gugpXUN49j2QQPcMj086r9+AkJ0FfUT8xjulKKBkkr9g==", + "dev": true, + "license": "MIT", + "peer": true, + "dependencies": { + "@vue/compiler-dom": "3.5.17", + "@vue/compiler-sfc": "3.5.17", + "@vue/runtime-dom": "3.5.17", + "@vue/server-renderer": "3.5.17", + "@vue/shared": "3.5.17" + }, + "peerDependencies": { + "typescript": "*" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/walker": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", @@ -16082,6 +17762,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zimmerframe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/zimmerframe/-/zimmerframe-1.1.2.tgz", + "integrity": "sha512-rAbqEGa8ovJy4pyBxZM70hg4pE6gDgaQ0Sl9M3enG3I0d6H4XSAM3GeNGLKnsBpuijUow064sf7ww1nutC5/3w==", + "dev": true, + "license": "MIT", + "peer": true + }, "node_modules/zod": { "version": "3.25.49", "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.49.tgz", diff --git a/package.json b/package.json index bdcffcd0..d5e47437 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "reformat": "prettier --write .", "generate": "./scripts/generate.sh", "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js --coverage --testPathIgnorePatterns=/tests/accuracy/", - "test:accuracy": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern tests/accuracy" + "test:accuracy": "node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern tests/accuracy", + "test:accuracy:c": "node --experimental-vm-modules node_modules/jest/bin/jest.js" }, "license": "Apache-2.0", "devDependencies": { @@ -45,6 +46,8 @@ "@types/simple-oauth2": "^5.0.7", "@types/yargs-parser": "^21.0.3", "ai": "^4.3.16", + "autoevals": "^0.0.130", + "braintrust": "^0.0.209", "eslint": "^9.24.0", "eslint-config-prettier": "^10.1.2", "eslint-plugin-jest": "^29.0.1", diff --git a/tests/accuracy/sdk/discover-mongodb-tools-stdio.ts b/tests/accuracy/sdk/discover-mongodb-tools-stdio.ts new file mode 100644 index 00000000..a049f474 --- /dev/null +++ b/tests/accuracy/sdk/discover-mongodb-tools-stdio.ts @@ -0,0 +1,32 @@ +import { Tool } from "@modelcontextprotocol/sdk/types.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; + +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import fs from "fs/promises"; + +export async function discoverMongoDBTools(): Promise { + let mcpClient: Client | undefined; + try { + const transport = new StdioClientTransport({ + command: "node", + args: ["dist/index.js"], + }); + + mcpClient = new Client( + { + name: "tool-discovery-client", + version: "0.0.0", + }, + { + capabilities: {}, + } + ); + await mcpClient.connect(transport); + + return (await mcpClient.listTools()).tools; + } finally { + await mcpClient?.close(); + } +} + +await fs.writeFile("mcp-tools.json", JSON.stringify(await discoverMongoDBTools(), null, 2)); diff --git a/tests/accuracy/sdk/discover-mongodb-tools.ts b/tests/accuracy/sdk/discover-mongodb-tools.ts new file mode 100644 index 00000000..fc4e9b80 --- /dev/null +++ b/tests/accuracy/sdk/discover-mongodb-tools.ts @@ -0,0 +1,59 @@ +import { Tool } from "@modelcontextprotocol/sdk/types.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; + +import { InMemoryTransport } from "../../integration/inMemoryTransport.js"; +import { defaultTestConfig } from "../../integration/helpers.js"; +import { Session } from "../../../src/session.js"; +import { Telemetry } from "../../../src/telemetry/telemetry.js"; +import { Server } from "../../../src/server.js"; + +export async function discoverMongoDBTools(): Promise { + let mcpClient: Client | undefined; + let mcpServer: Server | undefined; + try { + const serverTransport = new InMemoryTransport(); + const clientTransport = new InMemoryTransport(); + + await serverTransport.start(); + await clientTransport.start(); + + void serverTransport.output.pipeTo(clientTransport.input); + void clientTransport.output.pipeTo(serverTransport.input); + + const session = new Session({ + apiBaseUrl: defaultTestConfig.apiBaseUrl, + }); + + const telemetry = Telemetry.create(session, defaultTestConfig); + + mcpClient = new Client( + { + name: "tool-discovery-client", + version: "0.0.0", + }, + { + capabilities: {}, + } + ); + + mcpServer = new Server({ + session, + userConfig: defaultTestConfig, + telemetry, + mcpServer: new McpServer({ + name: "test-server", + version: "5.2.3", + }), + }); + + await mcpServer.connect(serverTransport); + await mcpClient.connect(clientTransport); + + return (await mcpClient.listTools()).tools; + } finally { + await mcpClient?.close(); + await mcpServer?.session?.close(); + await mcpServer?.close(); + } +} diff --git a/tests/accuracy/sdk/test-tools.ts b/tests/accuracy/sdk/test-tools.ts index 15bb0420..92057cad 100644 --- a/tests/accuracy/sdk/test-tools.ts +++ b/tests/accuracy/sdk/test-tools.ts @@ -1,15 +1,8 @@ import { JSONSchema7 } from "json-schema"; import { v4 as uuid } from "uuid"; import { Tool as VercelTool, Schema, tool as createVercelTool, jsonSchema } from "ai"; -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { CallToolResult, Tool } from "@modelcontextprotocol/sdk/types.js"; -import { InMemoryTransport } from "../../integration/inMemoryTransport.js"; -import { defaultTestConfig } from "../../integration/helpers.js"; -import { Session } from "../../../src/session.js"; -import { Telemetry } from "../../../src/telemetry/telemetry.js"; -import { Server } from "../../../src/server.js"; import { ToolCall } from "./accuracy-scorers.js"; type ToolResultGeneratorFn = (...parameters: unknown[]) => CallToolResult | Promise; @@ -88,53 +81,3 @@ export class TestTools { return vercelTools; } } - -export async function discoverMongoDBTools(): Promise { - let mcpClient: Client | undefined; - let mcpServer: Server | undefined; - try { - const serverTransport = new InMemoryTransport(); - const clientTransport = new InMemoryTransport(); - - await serverTransport.start(); - await clientTransport.start(); - - void serverTransport.output.pipeTo(clientTransport.input); - void clientTransport.output.pipeTo(serverTransport.input); - - const session = new Session({ - apiBaseUrl: defaultTestConfig.apiBaseUrl, - }); - - const telemetry = Telemetry.create(session, defaultTestConfig); - - mcpClient = new Client( - { - name: "tool-discovery-client", - version: "0.0.0", - }, - { - capabilities: {}, - } - ); - - mcpServer = new Server({ - session, - userConfig: defaultTestConfig, - telemetry, - mcpServer: new McpServer({ - name: "test-server", - version: "5.2.3", - }), - }); - - await mcpServer.connect(serverTransport); - await mcpClient.connect(clientTransport); - - return (await mcpClient.listTools()).tools; - } finally { - await mcpClient?.close(); - await mcpServer?.session?.close(); - await mcpServer?.close(); - } -} diff --git a/tests/evals/eval.ts b/tests/evals/eval.ts new file mode 100644 index 00000000..2e675fe9 --- /dev/null +++ b/tests/evals/eval.ts @@ -0,0 +1,41 @@ +import { Eval } from "braintrust"; +import { getAvailableModels, Model } from "../accuracy/sdk/models.js"; +import { TestTools } from "../accuracy/sdk/test-tools.js"; +import { getVercelToolCallingAgent } from "../accuracy/sdk/agent.js"; +import { parameterMatchingAccuracyScorer, toolCallingAccuracyScorer } from "../accuracy/sdk/accuracy-scorers.js"; +import { Tool } from "@modelcontextprotocol/sdk/types.js"; +import { listDatabasesTests } from "./tests/list-databases.eval.js"; + +// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-require-imports +const mcpTools = require("../../mcp-tools.json"); + +void Eval(listDatabasesTests.evalName, { + projectId: process.env.BRAIN_TRUST_PROJECT_ID, + data: listDatabasesTests.testConfigs.map((testConfig) => ({ + input: testConfig.prompt, + expected: testConfig.expectedToolCalls, + metadata: { + mockedTools: testConfig.mockedTools, + }, + })), + task: async (input, { metadata }) => { + const testTools = new TestTools(mcpTools as Tool[]); + testTools.mockTools(metadata.mockedTools); + const agent = getVercelToolCallingAgent(); + const models = getAvailableModels(); + const model = models[0] as Model; + const conversation = await agent.prompt(input, model, testTools.vercelAiTools()); + return { + conversation, + toolCalls: testTools.getToolCalls(), + }; + }, + scores: [ + function toolCallingAccuracy({ output, expected }): number { + return toolCallingAccuracyScorer(expected, output.toolCalls); + }, + function parameterPassingAccuracy({ output, expected }): number { + return parameterMatchingAccuracyScorer(expected, output.toolCalls); + }, + ], +}); diff --git a/tests/evals/tests/list-databases.eval.ts b/tests/evals/tests/list-databases.eval.ts new file mode 100644 index 00000000..e385f92a --- /dev/null +++ b/tests/evals/tests/list-databases.eval.ts @@ -0,0 +1,38 @@ +import { listDatabasesResponse } from "../../../src/tools/mongodb/metadata/listDatabases.js"; +import { AccuracyTestConfig } from "../../accuracy/sdk/describe-accuracy-tests.js"; + +function callsListDatabases(prompt: string): AccuracyTestConfig { + return { + injectConnectedAssumption: true, + prompt: prompt, + mockedTools: { + "list-databases": function listDatabases() { + return listDatabasesResponse([ + { + name: "db1", + sizeOnDisk: "1024", + }, + { + name: "db2", + sizeOnDisk: "2048", + }, + ]); + }, + }, + expectedToolCalls: [ + { + toolName: "list-databases", + parameters: {}, + }, + ], + }; +} + +export const listDatabasesTests = { + evalName: "should call list-databases tool", + testConfigs: [ + callsListDatabases("How many databases do I have?"), + callsListDatabases("List all the databases in my cluster."), + callsListDatabases("Is there a sample_mflix database in my cluster?"), + ], +};