diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49a043930..09f1636b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,7 +72,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' - name: Bootstrap run: ./scripts/bootstrap @@ -92,7 +92,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v4 with: - node-version: '18' + node-version: '20' - name: Install dependencies run: | yarn install diff --git a/.github/workflows/publish-jsr.yml b/.github/workflows/publish-jsr.yml index 1e46d6bfb..e74673c1f 100644 --- a/.github/workflows/publish-jsr.yml +++ b/.github/workflows/publish-jsr.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v3 with: - node-version: '18' + node-version: '20' - name: Install dependencies run: | diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 5a3711b53..0662a79c5 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Node uses: actions/setup-node@v3 with: - node-version: '18' + node-version: '20' - name: Install dependencies run: | diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 4e19f03d6..a279d9124 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "4.97.0" + ".": "4.98.0" } diff --git a/.stats.yml b/.stats.yml index 0c8278866..5f1bee851 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 97 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-0ee6b36cf3cc278cef4199a6aec5f7d530a6c1f17a74830037e96d50ca1edc50.yml -openapi_spec_hash: e8ec5f46bc0655b34f292422d58a60f6 -config_hash: d9b6b6e6bc85744663e300eebc482067 +configured_endpoints: 101 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-794a6ed3c3d3d77887564755168056af8a426b17cf1ec721e3a300503dc22a41.yml +openapi_spec_hash: 25a81c220713cd5b0bafc221d1dfa79a +config_hash: 0b768ed1b56c6d82816f0fa40dc4aaf5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa637742..2f1a39177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 4.98.0 (2025-05-08) + +Full Changelog: [v4.97.0...v4.98.0](https://github.com/openai/openai-node/compare/v4.97.0...v4.98.0) + +### Features + +* **api:** Add reinforcement fine-tuning api support ([4aa7a79](https://github.com/openai/openai-node/commit/4aa7a7954c63caa26cc1640ace56093fe1cafa04)) + + +### Chores + +* **ci:** bump node version for release workflows ([2961f63](https://github.com/openai/openai-node/commit/2961f63c4d5b8ae8efdf8ea6581aa83c6b0f722e)) +* **internal:** fix formatting ([91a44fe](https://github.com/openai/openai-node/commit/91a44fe11c0847dc50d48a03a8d409ac4bece37a)) + + +### Documentation + +* add examples to tsdocs ([7d841b7](https://github.com/openai/openai-node/commit/7d841b7f98eb542a398fb9de12056125e8d6cb22)) + ## 4.97.0 (2025-05-02) Full Changelog: [v4.96.2...v4.97.0](https://github.com/openai/openai-node/compare/v4.96.2...v4.97.0) diff --git a/api.md b/api.md index 49e6548a8..cad696e7e 100644 --- a/api.md +++ b/api.md @@ -207,6 +207,17 @@ Methods: # FineTuning +## Methods + +Types: + +- DpoHyperparameters +- DpoMethod +- ReinforcementHyperparameters +- ReinforcementMethod +- SupervisedHyperparameters +- SupervisedMethod + ## Jobs Types: @@ -224,6 +235,8 @@ Methods: - client.fineTuning.jobs.list({ ...params }) -> FineTuningJobsPage - client.fineTuning.jobs.cancel(fineTuningJobId) -> FineTuningJob - client.fineTuning.jobs.listEvents(fineTuningJobId, { ...params }) -> FineTuningJobEventsPage +- client.fineTuning.jobs.pause(fineTuningJobId) -> FineTuningJob +- client.fineTuning.jobs.resume(fineTuningJobId) -> FineTuningJob ### Checkpoints @@ -251,6 +264,33 @@ Methods: - client.fineTuning.checkpoints.permissions.retrieve(fineTunedModelCheckpoint, { ...params }) -> PermissionRetrieveResponse - client.fineTuning.checkpoints.permissions.del(fineTunedModelCheckpoint, permissionId) -> PermissionDeleteResponse +## Alpha + +### Graders + +Types: + +- GraderRunResponse +- GraderValidateResponse + +Methods: + +- client.fineTuning.alpha.graders.run({ ...params }) -> GraderRunResponse +- client.fineTuning.alpha.graders.validate({ ...params }) -> GraderValidateResponse + +# Graders + +## GraderModels + +Types: + +- LabelModelGrader +- MultiGrader +- PythonGrader +- ScoreModelGrader +- StringCheckGrader +- TextSimilarityGrader + # VectorStores Types: @@ -669,10 +709,7 @@ Methods: Types: - EvalCustomDataSourceConfig -- EvalLabelModelGrader - EvalStoredCompletionsDataSourceConfig -- EvalStringCheckGrader -- EvalTextSimilarityGrader - EvalCreateResponse - EvalRetrieveResponse - EvalUpdateResponse diff --git a/examples/azure/realtime/websocket.ts b/examples/azure/realtime/websocket.ts index 4175b4a71..91fe3b7b9 100644 --- a/examples/azure/realtime/websocket.ts +++ b/examples/azure/realtime/websocket.ts @@ -57,4 +57,4 @@ async function main() { rt.socket.addEventListener('close', () => console.log('\nConnection closed!')); } -main(); \ No newline at end of file +main(); diff --git a/examples/azure/realtime/ws.ts b/examples/azure/realtime/ws.ts index e86a79092..8b22aeef0 100644 --- a/examples/azure/realtime/ws.ts +++ b/examples/azure/realtime/ws.ts @@ -57,4 +57,4 @@ async function main() { rt.socket.on('close', () => console.log('\nConnection closed!')); } -main(); \ No newline at end of file +main(); diff --git a/examples/realtime/websocket.ts b/examples/realtime/websocket.ts index f1c46dd41..6fb4740af 100644 --- a/examples/realtime/websocket.ts +++ b/examples/realtime/websocket.ts @@ -45,4 +45,4 @@ async function main() { rt.socket.addEventListener('close', () => console.log('\nConnection closed!')); } -main(); \ No newline at end of file +main(); diff --git a/examples/realtime/ws.ts b/examples/realtime/ws.ts index 1ce6b2045..6cc950b76 100644 --- a/examples/realtime/ws.ts +++ b/examples/realtime/ws.ts @@ -45,4 +45,4 @@ async function main() { rt.socket.on('close', () => console.log('\nConnection closed!')); } -main(); \ No newline at end of file +main(); diff --git a/examples/tsconfig.json b/examples/tsconfig.json index 6c3477462..3c43903cf 100644 --- a/examples/tsconfig.json +++ b/examples/tsconfig.json @@ -1,3 +1,3 @@ { - "extends": "../tsconfig.json" + "extends": "../tsconfig.json" } diff --git a/jsr.json b/jsr.json index fd3ca4a41..25bbc9ac2 100644 --- a/jsr.json +++ b/jsr.json @@ -1,6 +1,6 @@ { "name": "@openai/openai", - "version": "4.97.0", + "version": "4.98.0", "exports": { ".": "./index.ts", "./helpers/zod": "./helpers/zod.ts", diff --git a/package.json b/package.json index 8b9281b35..d34efceb0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openai", - "version": "4.97.0", + "version": "4.98.0", "description": "The official TypeScript library for the OpenAI API", "author": "OpenAI ", "types": "dist/index.d.ts", diff --git a/src/index.ts b/src/index.ts index 9e8d7ce37..537c18f43 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,19 +71,17 @@ import { EvalCreateResponse, EvalCustomDataSourceConfig, EvalDeleteResponse, - EvalLabelModelGrader, EvalListParams, EvalListResponse, EvalListResponsesPage, EvalRetrieveResponse, EvalStoredCompletionsDataSourceConfig, - EvalStringCheckGrader, - EvalTextSimilarityGrader, EvalUpdateParams, EvalUpdateResponse, Evals, } from './resources/evals/evals'; import { FineTuning } from './resources/fine-tuning/fine-tuning'; +import { Graders } from './resources/graders/graders'; import { Responses } from './resources/responses/responses'; import { Upload, @@ -305,6 +303,7 @@ export class OpenAI extends Core.APIClient { moderations: API.Moderations = new API.Moderations(this); models: API.Models = new API.Models(this); fineTuning: API.FineTuning = new API.FineTuning(this); + graders: API.Graders = new API.Graders(this); vectorStores: API.VectorStores = new API.VectorStores(this); beta: API.Beta = new API.Beta(this); batches: API.Batches = new API.Batches(this); @@ -366,6 +365,7 @@ OpenAI.Moderations = Moderations; OpenAI.Models = Models; OpenAI.ModelsPage = ModelsPage; OpenAI.FineTuning = FineTuning; +OpenAI.Graders = Graders; OpenAI.VectorStores = VectorStores; OpenAI.VectorStoresPage = VectorStoresPage; OpenAI.VectorStoreSearchResponsesPage = VectorStoreSearchResponsesPage; @@ -487,6 +487,8 @@ export declare namespace OpenAI { export { FineTuning as FineTuning }; + export { Graders as Graders }; + export { VectorStores as VectorStores, type AutoFileChunkingStrategyParam as AutoFileChunkingStrategyParam, @@ -531,10 +533,7 @@ export declare namespace OpenAI { export { Evals as Evals, type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig, - type EvalLabelModelGrader as EvalLabelModelGrader, type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig, - type EvalStringCheckGrader as EvalStringCheckGrader, - type EvalTextSimilarityGrader as EvalTextSimilarityGrader, type EvalCreateResponse as EvalCreateResponse, type EvalRetrieveResponse as EvalRetrieveResponse, type EvalUpdateResponse as EvalUpdateResponse, diff --git a/src/resources/audio/speech.ts b/src/resources/audio/speech.ts index e218c8299..ccd37c092 100644 --- a/src/resources/audio/speech.ts +++ b/src/resources/audio/speech.ts @@ -7,6 +7,18 @@ import { type Response } from '../../_shims/index'; export class Speech extends APIResource { /** * Generates audio from the input text. + * + * @example + * ```ts + * const speech = await client.audio.speech.create({ + * input: 'input', + * model: 'string', + * voice: 'ash', + * }); + * + * const content = await speech.blob(); + * console.log(content); + * ``` */ create(body: SpeechCreateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/audio/speech', { diff --git a/src/resources/audio/transcriptions.ts b/src/resources/audio/transcriptions.ts index ba4fec6c5..8d563e0ba 100644 --- a/src/resources/audio/transcriptions.ts +++ b/src/resources/audio/transcriptions.ts @@ -9,6 +9,15 @@ import { Stream } from '../../streaming'; export class Transcriptions extends APIResource { /** * Transcribes audio into the input language. + * + * @example + * ```ts + * const transcription = + * await client.audio.transcriptions.create({ + * file: fs.createReadStream('speech.mp3'), + * model: 'gpt-4o-transcribe', + * }); + * ``` */ create( body: TranscriptionCreateParamsNonStreaming<'json' | undefined>, diff --git a/src/resources/audio/translations.ts b/src/resources/audio/translations.ts index df312f876..1edb71a7d 100644 --- a/src/resources/audio/translations.ts +++ b/src/resources/audio/translations.ts @@ -8,6 +8,14 @@ import * as TranscriptionsAPI from './transcriptions'; export class Translations extends APIResource { /** * Translates audio into English. + * + * @example + * ```ts + * const translation = await client.audio.translations.create({ + * file: fs.createReadStream('speech.mp3'), + * model: 'whisper-1', + * }); + * ``` */ create( body: TranslationCreateParams<'json' | undefined>, diff --git a/src/resources/beta/assistants.ts b/src/resources/beta/assistants.ts index 00a6ff2cf..95581bbc8 100644 --- a/src/resources/beta/assistants.ts +++ b/src/resources/beta/assistants.ts @@ -14,6 +14,13 @@ import { AssistantStream } from '../../lib/AssistantStream'; export class Assistants extends APIResource { /** * Create an assistant with a model and instructions. + * + * @example + * ```ts + * const assistant = await client.beta.assistants.create({ + * model: 'gpt-4o', + * }); + * ``` */ create(body: AssistantCreateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/assistants', { @@ -25,6 +32,13 @@ export class Assistants extends APIResource { /** * Retrieves an assistant. + * + * @example + * ```ts + * const assistant = await client.beta.assistants.retrieve( + * 'assistant_id', + * ); + * ``` */ retrieve(assistantId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/assistants/${assistantId}`, { @@ -35,6 +49,13 @@ export class Assistants extends APIResource { /** * Modifies an assistant. + * + * @example + * ```ts + * const assistant = await client.beta.assistants.update( + * 'assistant_id', + * ); + * ``` */ update( assistantId: string, @@ -50,6 +71,14 @@ export class Assistants extends APIResource { /** * Returns a list of assistants. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const assistant of client.beta.assistants.list()) { + * // ... + * } + * ``` */ list( query?: AssistantListParams, @@ -72,6 +101,13 @@ export class Assistants extends APIResource { /** * Delete an assistant. + * + * @example + * ```ts + * const assistantDeleted = await client.beta.assistants.del( + * 'assistant_id', + * ); + * ``` */ del(assistantId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.delete(`/assistants/${assistantId}`, { diff --git a/src/resources/beta/realtime/sessions.ts b/src/resources/beta/realtime/sessions.ts index 28a44431e..a55a2678c 100644 --- a/src/resources/beta/realtime/sessions.ts +++ b/src/resources/beta/realtime/sessions.ts @@ -12,6 +12,12 @@ export class Sessions extends APIResource { * It responds with a session object, plus a `client_secret` key which contains a * usable ephemeral API token that can be used to authenticate browser clients for * the Realtime API. + * + * @example + * ```ts + * const session = + * await client.beta.realtime.sessions.create(); + * ``` */ create(body: SessionCreateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/realtime/sessions', { diff --git a/src/resources/beta/realtime/transcription-sessions.ts b/src/resources/beta/realtime/transcription-sessions.ts index a54ec1125..61e58a8e8 100644 --- a/src/resources/beta/realtime/transcription-sessions.ts +++ b/src/resources/beta/realtime/transcription-sessions.ts @@ -12,6 +12,12 @@ export class TranscriptionSessions extends APIResource { * It responds with a session object, plus a `client_secret` key which contains a * usable ephemeral API token that can be used to authenticate browser clients for * the Realtime API. + * + * @example + * ```ts + * const transcriptionSession = + * await client.beta.realtime.transcriptionSessions.create(); + * ``` */ create( body: TranscriptionSessionCreateParams, diff --git a/src/resources/beta/threads/messages.ts b/src/resources/beta/threads/messages.ts index 29fd2b29f..c3834ebe6 100644 --- a/src/resources/beta/threads/messages.ts +++ b/src/resources/beta/threads/messages.ts @@ -10,6 +10,14 @@ import { CursorPage, type CursorPageParams } from '../../../pagination'; export class Messages extends APIResource { /** * Create a message. + * + * @example + * ```ts + * const message = await client.beta.threads.messages.create( + * 'thread_id', + * { content: 'string', role: 'user' }, + * ); + * ``` */ create( threadId: string, @@ -25,6 +33,14 @@ export class Messages extends APIResource { /** * Retrieve a message. + * + * @example + * ```ts + * const message = await client.beta.threads.messages.retrieve( + * 'thread_id', + * 'message_id', + * ); + * ``` */ retrieve(threadId: string, messageId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/threads/${threadId}/messages/${messageId}`, { @@ -35,6 +51,14 @@ export class Messages extends APIResource { /** * Modifies a message. + * + * @example + * ```ts + * const message = await client.beta.threads.messages.update( + * 'thread_id', + * 'message_id', + * ); + * ``` */ update( threadId: string, @@ -51,6 +75,16 @@ export class Messages extends APIResource { /** * Returns a list of messages for a given thread. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const message of client.beta.threads.messages.list( + * 'thread_id', + * )) { + * // ... + * } + * ``` */ list( threadId: string, @@ -75,6 +109,15 @@ export class Messages extends APIResource { /** * Deletes a message. + * + * @example + * ```ts + * const messageDeleted = + * await client.beta.threads.messages.del( + * 'thread_id', + * 'message_id', + * ); + * ``` */ del(threadId: string, messageId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.delete(`/threads/${threadId}/messages/${messageId}`, { diff --git a/src/resources/beta/threads/runs/runs.ts b/src/resources/beta/threads/runs/runs.ts index 15bfb4204..25356df3c 100644 --- a/src/resources/beta/threads/runs/runs.ts +++ b/src/resources/beta/threads/runs/runs.ts @@ -45,6 +45,14 @@ export class Runs extends APIResource { /** * Create a run. + * + * @example + * ```ts + * const run = await client.beta.threads.runs.create( + * 'thread_id', + * { assistant_id: 'assistant_id' }, + * ); + * ``` */ create( threadId: string, @@ -78,6 +86,14 @@ export class Runs extends APIResource { /** * Retrieves a run. + * + * @example + * ```ts + * const run = await client.beta.threads.runs.retrieve( + * 'thread_id', + * 'run_id', + * ); + * ``` */ retrieve(threadId: string, runId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/threads/${threadId}/runs/${runId}`, { @@ -88,6 +104,14 @@ export class Runs extends APIResource { /** * Modifies a run. + * + * @example + * ```ts + * const run = await client.beta.threads.runs.update( + * 'thread_id', + * 'run_id', + * ); + * ``` */ update( threadId: string, @@ -104,6 +128,16 @@ export class Runs extends APIResource { /** * Returns a list of runs belonging to a thread. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const run of client.beta.threads.runs.list( + * 'thread_id', + * )) { + * // ... + * } + * ``` */ list( threadId: string, @@ -128,6 +162,14 @@ export class Runs extends APIResource { /** * Cancels a run that is `in_progress`. + * + * @example + * ```ts + * const run = await client.beta.threads.runs.cancel( + * 'thread_id', + * 'run_id', + * ); + * ``` */ cancel(threadId: string, runId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.post(`/threads/${threadId}/runs/${runId}/cancel`, { @@ -229,6 +271,16 @@ export class Runs extends APIResource { * `submit_tool_outputs`, this endpoint can be used to submit the outputs from the * tool calls once they're all completed. All outputs must be submitted in a single * request. + * + * @example + * ```ts + * const run = + * await client.beta.threads.runs.submitToolOutputs( + * 'thread_id', + * 'run_id', + * { tool_outputs: [{}] }, + * ); + * ``` */ submitToolOutputs( threadId: string, diff --git a/src/resources/beta/threads/runs/steps.ts b/src/resources/beta/threads/runs/steps.ts index c491b4e83..abd8d40ed 100644 --- a/src/resources/beta/threads/runs/steps.ts +++ b/src/resources/beta/threads/runs/steps.ts @@ -10,6 +10,16 @@ import { CursorPage, type CursorPageParams } from '../../../../pagination'; export class Steps extends APIResource { /** * Retrieves a run step. + * + * @example + * ```ts + * const runStep = + * await client.beta.threads.runs.steps.retrieve( + * 'thread_id', + * 'run_id', + * 'step_id', + * ); + * ``` */ retrieve( threadId: string, @@ -43,6 +53,17 @@ export class Steps extends APIResource { /** * Returns a list of run steps belonging to a run. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const runStep of client.beta.threads.runs.steps.list( + * 'thread_id', + * 'run_id', + * )) { + * // ... + * } + * ``` */ list( threadId: string, diff --git a/src/resources/beta/threads/threads.ts b/src/resources/beta/threads/threads.ts index 2e5ab1cc8..c0c6bc8e4 100644 --- a/src/resources/beta/threads/threads.ts +++ b/src/resources/beta/threads/threads.ts @@ -73,6 +73,11 @@ export class Threads extends APIResource { /** * Create a thread. + * + * @example + * ```ts + * const thread = await client.beta.threads.create(); + * ``` */ create(body?: ThreadCreateParams, options?: Core.RequestOptions): Core.APIPromise; create(options?: Core.RequestOptions): Core.APIPromise; @@ -92,6 +97,13 @@ export class Threads extends APIResource { /** * Retrieves a thread. + * + * @example + * ```ts + * const thread = await client.beta.threads.retrieve( + * 'thread_id', + * ); + * ``` */ retrieve(threadId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/threads/${threadId}`, { @@ -102,6 +114,13 @@ export class Threads extends APIResource { /** * Modifies a thread. + * + * @example + * ```ts + * const thread = await client.beta.threads.update( + * 'thread_id', + * ); + * ``` */ update(threadId: string, body: ThreadUpdateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post(`/threads/${threadId}`, { @@ -113,6 +132,13 @@ export class Threads extends APIResource { /** * Delete a thread. + * + * @example + * ```ts + * const threadDeleted = await client.beta.threads.del( + * 'thread_id', + * ); + * ``` */ del(threadId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.delete(`/threads/${threadId}`, { @@ -123,6 +149,13 @@ export class Threads extends APIResource { /** * Create a thread and run it in one request. + * + * @example + * ```ts + * const run = await client.beta.threads.createAndRun({ + * assistant_id: 'assistant_id', + * }); + * ``` */ createAndRun( body: ThreadCreateAndRunParamsNonStreaming, diff --git a/src/resources/chat/completions/completions.ts b/src/resources/chat/completions/completions.ts index 251020337..6481f8e0f 100644 --- a/src/resources/chat/completions/completions.ts +++ b/src/resources/chat/completions/completions.ts @@ -33,6 +33,16 @@ export class Completions extends APIResource { * supported for reasoning models are noted below. For the current state of * unsupported parameters in reasoning models, * [refer to the reasoning guide](https://platform.openai.com/docs/guides/reasoning). + * + * @example + * ```ts + * const chatCompletion = await client.chat.completions.create( + * { + * messages: [{ content: 'string', role: 'developer' }], + * model: 'gpt-4o', + * }, + * ); + * ``` */ create( body: ChatCompletionCreateParamsNonStreaming, @@ -58,6 +68,12 @@ export class Completions extends APIResource { /** * Get a stored chat completion. Only Chat Completions that have been created with * the `store` parameter set to `true` will be returned. + * + * @example + * ```ts + * const chatCompletion = + * await client.chat.completions.retrieve('completion_id'); + * ``` */ retrieve(completionId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/chat/completions/${completionId}`, options); @@ -67,6 +83,14 @@ export class Completions extends APIResource { * Modify a stored chat completion. Only Chat Completions that have been created * with the `store` parameter set to `true` can be modified. Currently, the only * supported modification is to update the `metadata` field. + * + * @example + * ```ts + * const chatCompletion = await client.chat.completions.update( + * 'completion_id', + * { metadata: { foo: 'string' } }, + * ); + * ``` */ update( completionId: string, @@ -79,6 +103,14 @@ export class Completions extends APIResource { /** * List stored Chat Completions. Only Chat Completions that have been stored with * the `store` parameter set to `true` will be returned. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const chatCompletion of client.chat.completions.list()) { + * // ... + * } + * ``` */ list( query?: ChatCompletionListParams, @@ -98,6 +130,12 @@ export class Completions extends APIResource { /** * Delete a stored chat completion. Only Chat Completions that have been created * with the `store` parameter set to `true` can be deleted. + * + * @example + * ```ts + * const chatCompletionDeleted = + * await client.chat.completions.del('completion_id'); + * ``` */ del(completionId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.delete(`/chat/completions/${completionId}`, options); diff --git a/src/resources/chat/completions/messages.ts b/src/resources/chat/completions/messages.ts index 519a33aff..ab3eb73f6 100644 --- a/src/resources/chat/completions/messages.ts +++ b/src/resources/chat/completions/messages.ts @@ -11,6 +11,16 @@ export class Messages extends APIResource { /** * Get the messages in a stored chat completion. Only Chat Completions that have * been created with the `store` parameter set to `true` will be returned. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const chatCompletionStoreMessage of client.chat.completions.messages.list( + * 'completion_id', + * )) { + * // ... + * } + * ``` */ list( completionId: string, diff --git a/src/resources/completions.ts b/src/resources/completions.ts index 5cbec5e3c..07cb49ed9 100644 --- a/src/resources/completions.ts +++ b/src/resources/completions.ts @@ -10,6 +10,14 @@ import { Stream } from '../streaming'; export class Completions extends APIResource { /** * Creates a completion for the provided prompt and parameters. + * + * @example + * ```ts + * const completion = await client.completions.create({ + * model: 'string', + * prompt: 'This is a test.', + * }); + * ``` */ create(body: CompletionCreateParamsNonStreaming, options?: Core.RequestOptions): APIPromise; create( diff --git a/src/resources/embeddings.ts b/src/resources/embeddings.ts index a4be9ca3c..cc040abff 100644 --- a/src/resources/embeddings.ts +++ b/src/resources/embeddings.ts @@ -6,6 +6,15 @@ import * as Core from '../core'; export class Embeddings extends APIResource { /** * Creates an embedding vector representing the input text. + * + * @example + * ```ts + * const createEmbeddingResponse = + * await client.embeddings.create({ + * input: 'The quick brown fox jumped over the lazy dog', + * model: 'text-embedding-3-small', + * }); + * ``` */ create( body: EmbeddingCreateParams, diff --git a/src/resources/evals/evals.ts b/src/resources/evals/evals.ts index caef7acc0..0f7166df4 100644 --- a/src/resources/evals/evals.ts +++ b/src/resources/evals/evals.ts @@ -4,6 +4,7 @@ import { APIResource } from '../../resource'; import { isRequestOptions } from '../../core'; import * as Core from '../../core'; import * as Shared from '../shared'; +import * as GraderModelsAPI from '../graders/grader-models'; import * as ResponsesAPI from '../responses/responses'; import * as RunsAPI from './runs/runs'; import { @@ -103,83 +104,6 @@ export interface EvalCustomDataSourceConfig { type: 'custom'; } -/** - * A LabelModelGrader object which uses a model to assign labels to each item in - * the evaluation. - */ -export interface EvalLabelModelGrader { - input: Array; - - /** - * The labels to assign to each item in the evaluation. - */ - labels: Array; - - /** - * The model to use for the evaluation. Must support structured outputs. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The labels that indicate a passing result. Must be a subset of labels. - */ - passing_labels: Array; - - /** - * The object type, which is always `label_model`. - */ - type: 'label_model'; -} - -export namespace EvalLabelModelGrader { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } -} - /** * A StoredCompletionsDataSourceConfig which specifies the metadata property of * your stored completions query. This is usually metadata like `usecase=chatbot` @@ -210,83 +134,6 @@ export interface EvalStoredCompletionsDataSourceConfig { metadata?: Shared.Metadata | null; } -/** - * A StringCheckGrader object that performs a string comparison between input and - * reference using a specified operation. - */ -export interface EvalStringCheckGrader { - /** - * The input text. This may include template strings. - */ - input: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. - */ - operation: 'eq' | 'ne' | 'like' | 'ilike'; - - /** - * The reference text. This may include template strings. - */ - reference: string; - - /** - * The object type, which is always `string_check`. - */ - type: 'string_check'; -} - -/** - * A TextSimilarityGrader object which grades text based on similarity metrics. - */ -export interface EvalTextSimilarityGrader { - /** - * The evaluation metric to use. One of `fuzzy_match`, `bleu`, `gleu`, `meteor`, - * `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, or `rouge_l`. - */ - evaluation_metric: - | 'fuzzy_match' - | 'bleu' - | 'gleu' - | 'meteor' - | 'rouge_1' - | 'rouge_2' - | 'rouge_3' - | 'rouge_4' - | 'rouge_5' - | 'rouge_l'; - - /** - * The text being graded. - */ - input: string; - - /** - * A float score where a value greater than or equal indicates a passing grade. - */ - pass_threshold: number; - - /** - * The text being graded against. - */ - reference: string; - - /** - * The type of grader. - */ - type: 'text_similarity'; - - /** - * The name of the grader. - */ - name?: string; -} - /** * An Eval object with a data source config and testing criteria. An Eval * represents a task to be done for your LLM integration. Like: @@ -335,39 +182,29 @@ export interface EvalCreateResponse { * A list of testing criteria. */ testing_criteria: Array< - | EvalLabelModelGrader - | EvalStringCheckGrader - | EvalTextSimilarityGrader - | EvalCreateResponse.Python - | EvalCreateResponse.ScoreModel + | GraderModelsAPI.LabelModelGrader + | GraderModelsAPI.StringCheckGrader + | EvalCreateResponse.EvalGraderTextSimilarity + | EvalCreateResponse.EvalGraderPython + | EvalCreateResponse.EvalGraderScoreModel >; } export namespace EvalCreateResponse { /** - * A PythonGrader object that runs a python script on the input. + * A TextSimilarityGrader object which grades text based on similarity metrics. */ - export interface Python { - /** - * The name of the grader. - */ - name: string; - + export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader { /** - * The source code of the python script. - */ - source: string; - - /** - * The object type, which is always `python`. - */ - type: 'python'; - - /** - * The image tag to use for the python script. + * The threshold for the score. */ - image_tag?: string; + pass_threshold: number; + } + /** + * A PythonGrader object that runs a python script on the input. + */ + export interface EvalGraderPython extends GraderModelsAPI.PythonGrader { /** * The threshold for the score. */ @@ -377,85 +214,11 @@ export namespace EvalCreateResponse { /** * A ScoreModelGrader object that uses a model to assign a score to the input. */ - export interface ScoreModel { - /** - * The input text. This may include template strings. - */ - input: Array; - - /** - * The model to use for the evaluation. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The object type, which is always `score_model`. - */ - type: 'score_model'; - + export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader { /** * The threshold for the score. */ pass_threshold?: number; - - /** - * The range of the score. Defaults to `[0, 1]`. - */ - range?: Array; - - /** - * The sampling parameters for the model. - */ - sampling_params?: unknown; - } - - export namespace ScoreModel { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } } } @@ -507,39 +270,29 @@ export interface EvalRetrieveResponse { * A list of testing criteria. */ testing_criteria: Array< - | EvalLabelModelGrader - | EvalStringCheckGrader - | EvalTextSimilarityGrader - | EvalRetrieveResponse.Python - | EvalRetrieveResponse.ScoreModel + | GraderModelsAPI.LabelModelGrader + | GraderModelsAPI.StringCheckGrader + | EvalRetrieveResponse.EvalGraderTextSimilarity + | EvalRetrieveResponse.EvalGraderPython + | EvalRetrieveResponse.EvalGraderScoreModel >; } export namespace EvalRetrieveResponse { /** - * A PythonGrader object that runs a python script on the input. + * A TextSimilarityGrader object which grades text based on similarity metrics. */ - export interface Python { + export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader { /** - * The name of the grader. - */ - name: string; - - /** - * The source code of the python script. - */ - source: string; - - /** - * The object type, which is always `python`. - */ - type: 'python'; - - /** - * The image tag to use for the python script. + * The threshold for the score. */ - image_tag?: string; + pass_threshold: number; + } + /** + * A PythonGrader object that runs a python script on the input. + */ + export interface EvalGraderPython extends GraderModelsAPI.PythonGrader { /** * The threshold for the score. */ @@ -549,85 +302,11 @@ export namespace EvalRetrieveResponse { /** * A ScoreModelGrader object that uses a model to assign a score to the input. */ - export interface ScoreModel { - /** - * The input text. This may include template strings. - */ - input: Array; - - /** - * The model to use for the evaluation. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The object type, which is always `score_model`. - */ - type: 'score_model'; - + export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader { /** * The threshold for the score. */ pass_threshold?: number; - - /** - * The range of the score. Defaults to `[0, 1]`. - */ - range?: Array; - - /** - * The sampling parameters for the model. - */ - sampling_params?: unknown; - } - - export namespace ScoreModel { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } } } @@ -679,39 +358,29 @@ export interface EvalUpdateResponse { * A list of testing criteria. */ testing_criteria: Array< - | EvalLabelModelGrader - | EvalStringCheckGrader - | EvalTextSimilarityGrader - | EvalUpdateResponse.Python - | EvalUpdateResponse.ScoreModel + | GraderModelsAPI.LabelModelGrader + | GraderModelsAPI.StringCheckGrader + | EvalUpdateResponse.EvalGraderTextSimilarity + | EvalUpdateResponse.EvalGraderPython + | EvalUpdateResponse.EvalGraderScoreModel >; } export namespace EvalUpdateResponse { /** - * A PythonGrader object that runs a python script on the input. + * A TextSimilarityGrader object which grades text based on similarity metrics. */ - export interface Python { - /** - * The name of the grader. - */ - name: string; - - /** - * The source code of the python script. - */ - source: string; - - /** - * The object type, which is always `python`. - */ - type: 'python'; - + export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader { /** - * The image tag to use for the python script. + * The threshold for the score. */ - image_tag?: string; + pass_threshold: number; + } + /** + * A PythonGrader object that runs a python script on the input. + */ + export interface EvalGraderPython extends GraderModelsAPI.PythonGrader { /** * The threshold for the score. */ @@ -721,85 +390,11 @@ export namespace EvalUpdateResponse { /** * A ScoreModelGrader object that uses a model to assign a score to the input. */ - export interface ScoreModel { - /** - * The input text. This may include template strings. - */ - input: Array; - - /** - * The model to use for the evaluation. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The object type, which is always `score_model`. - */ - type: 'score_model'; - + export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader { /** * The threshold for the score. */ pass_threshold?: number; - - /** - * The range of the score. Defaults to `[0, 1]`. - */ - range?: Array; - - /** - * The sampling parameters for the model. - */ - sampling_params?: unknown; - } - - export namespace ScoreModel { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } } } @@ -851,39 +446,29 @@ export interface EvalListResponse { * A list of testing criteria. */ testing_criteria: Array< - | EvalLabelModelGrader - | EvalStringCheckGrader - | EvalTextSimilarityGrader - | EvalListResponse.Python - | EvalListResponse.ScoreModel + | GraderModelsAPI.LabelModelGrader + | GraderModelsAPI.StringCheckGrader + | EvalListResponse.EvalGraderTextSimilarity + | EvalListResponse.EvalGraderPython + | EvalListResponse.EvalGraderScoreModel >; } export namespace EvalListResponse { /** - * A PythonGrader object that runs a python script on the input. + * A TextSimilarityGrader object which grades text based on similarity metrics. */ - export interface Python { - /** - * The name of the grader. - */ - name: string; - - /** - * The source code of the python script. - */ - source: string; - + export interface EvalGraderTextSimilarity extends GraderModelsAPI.TextSimilarityGrader { /** - * The object type, which is always `python`. - */ - type: 'python'; - - /** - * The image tag to use for the python script. + * The threshold for the score. */ - image_tag?: string; + pass_threshold: number; + } + /** + * A PythonGrader object that runs a python script on the input. + */ + export interface EvalGraderPython extends GraderModelsAPI.PythonGrader { /** * The threshold for the score. */ @@ -893,85 +478,11 @@ export namespace EvalListResponse { /** * A ScoreModelGrader object that uses a model to assign a score to the input. */ - export interface ScoreModel { - /** - * The input text. This may include template strings. - */ - input: Array; - - /** - * The model to use for the evaluation. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The object type, which is always `score_model`. - */ - type: 'score_model'; - + export interface EvalGraderScoreModel extends GraderModelsAPI.ScoreModelGrader { /** * The threshold for the score. */ pass_threshold?: number; - - /** - * The range of the score. Defaults to `[0, 1]`. - */ - range?: Array; - - /** - * The sampling parameters for the model. - */ - sampling_params?: unknown; - } - - export namespace ScoreModel { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } } } @@ -987,15 +498,15 @@ export interface EvalCreateParams { /** * The configuration for the data source used for the evaluation runs. */ - data_source_config: EvalCreateParams.Custom | EvalCreateParams.Logs; + data_source_config: EvalCreateParams.Custom | EvalCreateParams.StoredCompletions; /** * A list of graders for all eval runs in this group. */ testing_criteria: Array< | EvalCreateParams.LabelModel - | EvalStringCheckGrader - | EvalTextSimilarityGrader + | GraderModelsAPI.StringCheckGrader + | EvalCreateParams.TextSimilarity | EvalCreateParams.Python | EvalCreateParams.ScoreModel >; @@ -1048,14 +559,14 @@ export namespace EvalCreateParams { * completions query. This is usually metadata like `usecase=chatbot` or * `prompt-version=v2`, etc. */ - export interface Logs { + export interface StoredCompletions { /** - * The type of data source. Always `logs`. + * The type of data source. Always `stored_completions`. */ - type: 'logs'; + type: 'stored_completions'; /** - * Metadata filters for the logs data source. + * Metadata filters for the stored completions data source. */ metadata?: Record; } @@ -1154,29 +665,19 @@ export namespace EvalCreateParams { } /** - * A PythonGrader object that runs a python script on the input. + * A TextSimilarityGrader object which grades text based on similarity metrics. */ - export interface Python { + export interface TextSimilarity extends GraderModelsAPI.TextSimilarityGrader { /** - * The name of the grader. - */ - name: string; - - /** - * The source code of the python script. - */ - source: string; - - /** - * The object type, which is always `python`. - */ - type: 'python'; - - /** - * The image tag to use for the python script. + * The threshold for the score. */ - image_tag?: string; + pass_threshold: number; + } + /** + * A PythonGrader object that runs a python script on the input. + */ + export interface Python extends GraderModelsAPI.PythonGrader { /** * The threshold for the score. */ @@ -1186,85 +687,11 @@ export namespace EvalCreateParams { /** * A ScoreModelGrader object that uses a model to assign a score to the input. */ - export interface ScoreModel { - /** - * The input text. This may include template strings. - */ - input: Array; - - /** - * The model to use for the evaluation. - */ - model: string; - - /** - * The name of the grader. - */ - name: string; - - /** - * The object type, which is always `score_model`. - */ - type: 'score_model'; - + export interface ScoreModel extends GraderModelsAPI.ScoreModelGrader { /** * The threshold for the score. */ pass_threshold?: number; - - /** - * The range of the score. Defaults to `[0, 1]`. - */ - range?: Array; - - /** - * The sampling parameters for the model. - */ - sampling_params?: unknown; - } - - export namespace ScoreModel { - /** - * A message input to the model with a role indicating instruction following - * hierarchy. Instructions given with the `developer` or `system` role take - * precedence over instructions given with the `user` role. Messages with the - * `assistant` role are presumed to have been generated by the model in previous - * interactions. - */ - export interface Input { - /** - * Text inputs to the model - can contain template strings. - */ - content: string | ResponsesAPI.ResponseInputText | Input.OutputText; - - /** - * The role of the message input. One of `user`, `assistant`, `system`, or - * `developer`. - */ - role: 'user' | 'assistant' | 'system' | 'developer'; - - /** - * The type of the message input. Always `message`. - */ - type?: 'message'; - } - - export namespace Input { - /** - * A text output from the model. - */ - export interface OutputText { - /** - * The text output from the model. - */ - text: string; - - /** - * The type of the output text. Always `output_text`. - */ - type: 'output_text'; - } - } } } @@ -1306,10 +733,7 @@ Evals.RunListResponsesPage = RunListResponsesPage; export declare namespace Evals { export { type EvalCustomDataSourceConfig as EvalCustomDataSourceConfig, - type EvalLabelModelGrader as EvalLabelModelGrader, type EvalStoredCompletionsDataSourceConfig as EvalStoredCompletionsDataSourceConfig, - type EvalStringCheckGrader as EvalStringCheckGrader, - type EvalTextSimilarityGrader as EvalTextSimilarityGrader, type EvalCreateResponse as EvalCreateResponse, type EvalRetrieveResponse as EvalRetrieveResponse, type EvalUpdateResponse as EvalUpdateResponse, diff --git a/src/resources/evals/index.ts b/src/resources/evals/index.ts index a246fe4e7..b2627fbf3 100644 --- a/src/resources/evals/index.ts +++ b/src/resources/evals/index.ts @@ -4,10 +4,7 @@ export { EvalListResponsesPage, Evals, type EvalCustomDataSourceConfig, - type EvalLabelModelGrader, type EvalStoredCompletionsDataSourceConfig, - type EvalStringCheckGrader, - type EvalTextSimilarityGrader, type EvalCreateResponse, type EvalRetrieveResponse, type EvalUpdateResponse, diff --git a/src/resources/fine-tuning/alpha.ts b/src/resources/fine-tuning/alpha.ts new file mode 100644 index 000000000..446b6431e --- /dev/null +++ b/src/resources/fine-tuning/alpha.ts @@ -0,0 +1,3 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +export * from './alpha/index'; diff --git a/src/resources/fine-tuning/alpha/alpha.ts b/src/resources/fine-tuning/alpha/alpha.ts new file mode 100644 index 000000000..77d695195 --- /dev/null +++ b/src/resources/fine-tuning/alpha/alpha.ts @@ -0,0 +1,27 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../../../resource'; +import * as GradersAPI from './graders'; +import { + GraderRunParams, + GraderRunResponse, + GraderValidateParams, + GraderValidateResponse, + Graders, +} from './graders'; + +export class Alpha extends APIResource { + graders: GradersAPI.Graders = new GradersAPI.Graders(this._client); +} + +Alpha.Graders = Graders; + +export declare namespace Alpha { + export { + Graders as Graders, + type GraderRunResponse as GraderRunResponse, + type GraderValidateResponse as GraderValidateResponse, + type GraderRunParams as GraderRunParams, + type GraderValidateParams as GraderValidateParams, + }; +} diff --git a/src/resources/fine-tuning/alpha/graders.ts b/src/resources/fine-tuning/alpha/graders.ts new file mode 100644 index 000000000..a9ef57f71 --- /dev/null +++ b/src/resources/fine-tuning/alpha/graders.ts @@ -0,0 +1,168 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../../../resource'; +import * as Core from '../../../core'; +import * as GraderModelsAPI from '../../graders/grader-models'; + +export class Graders extends APIResource { + /** + * Run a grader. + * + * @example + * ```ts + * const response = await client.fineTuning.alpha.graders.run({ + * grader: { + * input: 'input', + * name: 'name', + * operation: 'eq', + * reference: 'reference', + * type: 'string_check', + * }, + * model_sample: 'model_sample', + * reference_answer: 'string', + * }); + * ``` + */ + run(body: GraderRunParams, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post('/fine_tuning/alpha/graders/run', { body, ...options }); + } + + /** + * Validate a grader. + * + * @example + * ```ts + * const response = + * await client.fineTuning.alpha.graders.validate({ + * grader: { + * input: 'input', + * name: 'name', + * operation: 'eq', + * reference: 'reference', + * type: 'string_check', + * }, + * }); + * ``` + */ + validate( + body: GraderValidateParams, + options?: Core.RequestOptions, + ): Core.APIPromise { + return this._client.post('/fine_tuning/alpha/graders/validate', { body, ...options }); + } +} + +export interface GraderRunResponse { + metadata: GraderRunResponse.Metadata; + + model_grader_token_usage_per_model: Record; + + reward: number; + + sub_rewards: Record; +} + +export namespace GraderRunResponse { + export interface Metadata { + errors: Metadata.Errors; + + execution_time: number; + + name: string; + + sampled_model_name: string | null; + + scores: Record; + + token_usage: number | null; + + type: string; + } + + export namespace Metadata { + export interface Errors { + formula_parse_error: boolean; + + invalid_variable_error: boolean; + + model_grader_parse_error: boolean; + + model_grader_refusal_error: boolean; + + model_grader_server_error: boolean; + + model_grader_server_error_details: string | null; + + other_error: boolean; + + python_grader_runtime_error: boolean; + + python_grader_runtime_error_details: string | null; + + python_grader_server_error: boolean; + + python_grader_server_error_type: string | null; + + sample_parse_error: boolean; + + truncated_observation_error: boolean; + + unresponsive_reward_error: boolean; + } + } +} + +export interface GraderValidateResponse { + /** + * The grader used for the fine-tuning job. + */ + grader?: + | GraderModelsAPI.StringCheckGrader + | GraderModelsAPI.TextSimilarityGrader + | GraderModelsAPI.PythonGrader + | GraderModelsAPI.ScoreModelGrader + | GraderModelsAPI.MultiGrader; +} + +export interface GraderRunParams { + /** + * The grader used for the fine-tuning job. + */ + grader: + | GraderModelsAPI.StringCheckGrader + | GraderModelsAPI.TextSimilarityGrader + | GraderModelsAPI.PythonGrader + | GraderModelsAPI.ScoreModelGrader + | GraderModelsAPI.MultiGrader; + + /** + * The model sample to be evaluated. + */ + model_sample: string; + + /** + * The reference answer for the evaluation. + */ + reference_answer: string | unknown | Array | number; +} + +export interface GraderValidateParams { + /** + * The grader used for the fine-tuning job. + */ + grader: + | GraderModelsAPI.StringCheckGrader + | GraderModelsAPI.TextSimilarityGrader + | GraderModelsAPI.PythonGrader + | GraderModelsAPI.ScoreModelGrader + | GraderModelsAPI.MultiGrader; +} + +export declare namespace Graders { + export { + type GraderRunResponse as GraderRunResponse, + type GraderValidateResponse as GraderValidateResponse, + type GraderRunParams as GraderRunParams, + type GraderValidateParams as GraderValidateParams, + }; +} diff --git a/src/resources/fine-tuning/alpha/index.ts b/src/resources/fine-tuning/alpha/index.ts new file mode 100644 index 000000000..47b229bc3 --- /dev/null +++ b/src/resources/fine-tuning/alpha/index.ts @@ -0,0 +1,10 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +export { Alpha } from './alpha'; +export { + Graders, + type GraderRunResponse, + type GraderValidateResponse, + type GraderRunParams, + type GraderValidateParams, +} from './graders'; diff --git a/src/resources/fine-tuning/checkpoints/permissions.ts b/src/resources/fine-tuning/checkpoints/permissions.ts index e808b2001..dc25bab7f 100644 --- a/src/resources/fine-tuning/checkpoints/permissions.ts +++ b/src/resources/fine-tuning/checkpoints/permissions.ts @@ -11,6 +11,17 @@ export class Permissions extends APIResource { * * This enables organization owners to share fine-tuned models with other projects * in their organization. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const permissionCreateResponse of client.fineTuning.checkpoints.permissions.create( + * 'ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd', + * { project_ids: ['string'] }, + * )) { + * // ... + * } + * ``` */ create( fineTunedModelCheckpoint: string, @@ -29,6 +40,14 @@ export class Permissions extends APIResource { * * Organization owners can use this endpoint to view all permissions for a * fine-tuned model checkpoint. + * + * @example + * ```ts + * const permission = + * await client.fineTuning.checkpoints.permissions.retrieve( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * ); + * ``` */ retrieve( fineTunedModelCheckpoint: string, @@ -58,6 +77,15 @@ export class Permissions extends APIResource { * * Organization owners can use this endpoint to delete a permission for a * fine-tuned model checkpoint. + * + * @example + * ```ts + * const permission = + * await client.fineTuning.checkpoints.permissions.del( + * 'ft:gpt-4o-mini-2024-07-18:org:weather:B7R9VjQd', + * 'cp_zc4Q7MP6XxulcVzj4MZdwsAB', + * ); + * ``` */ del( fineTunedModelCheckpoint: string, diff --git a/src/resources/fine-tuning/fine-tuning.ts b/src/resources/fine-tuning/fine-tuning.ts index 9b0a01992..8fb54983b 100644 --- a/src/resources/fine-tuning/fine-tuning.ts +++ b/src/resources/fine-tuning/fine-tuning.ts @@ -1,6 +1,18 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. import { APIResource } from '../../resource'; +import * as MethodsAPI from './methods'; +import { + DpoHyperparameters, + DpoMethod, + Methods, + ReinforcementHyperparameters, + ReinforcementMethod, + SupervisedHyperparameters, + SupervisedMethod, +} from './methods'; +import * as AlphaAPI from './alpha/alpha'; +import { Alpha } from './alpha/alpha'; import * as CheckpointsAPI from './checkpoints/checkpoints'; import { Checkpoints } from './checkpoints/checkpoints'; import * as JobsAPI from './jobs/jobs'; @@ -19,16 +31,30 @@ import { } from './jobs/jobs'; export class FineTuning extends APIResource { + methods: MethodsAPI.Methods = new MethodsAPI.Methods(this._client); jobs: JobsAPI.Jobs = new JobsAPI.Jobs(this._client); checkpoints: CheckpointsAPI.Checkpoints = new CheckpointsAPI.Checkpoints(this._client); + alpha: AlphaAPI.Alpha = new AlphaAPI.Alpha(this._client); } +FineTuning.Methods = Methods; FineTuning.Jobs = Jobs; FineTuning.FineTuningJobsPage = FineTuningJobsPage; FineTuning.FineTuningJobEventsPage = FineTuningJobEventsPage; FineTuning.Checkpoints = Checkpoints; +FineTuning.Alpha = Alpha; export declare namespace FineTuning { + export { + Methods as Methods, + type DpoHyperparameters as DpoHyperparameters, + type DpoMethod as DpoMethod, + type ReinforcementHyperparameters as ReinforcementHyperparameters, + type ReinforcementMethod as ReinforcementMethod, + type SupervisedHyperparameters as SupervisedHyperparameters, + type SupervisedMethod as SupervisedMethod, + }; + export { Jobs as Jobs, type FineTuningJob as FineTuningJob, @@ -44,4 +70,6 @@ export declare namespace FineTuning { }; export { Checkpoints as Checkpoints }; + + export { Alpha as Alpha }; } diff --git a/src/resources/fine-tuning/index.ts b/src/resources/fine-tuning/index.ts index d23161c62..878ac402d 100644 --- a/src/resources/fine-tuning/index.ts +++ b/src/resources/fine-tuning/index.ts @@ -1,5 +1,6 @@ // File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. +export { Alpha } from './alpha/index'; export { Checkpoints } from './checkpoints/index'; export { FineTuning } from './fine-tuning'; export { @@ -15,3 +16,12 @@ export { type JobListParams, type JobListEventsParams, } from './jobs/index'; +export { + Methods, + type DpoHyperparameters, + type DpoMethod, + type ReinforcementHyperparameters, + type ReinforcementMethod, + type SupervisedHyperparameters, + type SupervisedMethod, +} from './methods'; diff --git a/src/resources/fine-tuning/jobs/checkpoints.ts b/src/resources/fine-tuning/jobs/checkpoints.ts index b3018ac5f..10902e715 100644 --- a/src/resources/fine-tuning/jobs/checkpoints.ts +++ b/src/resources/fine-tuning/jobs/checkpoints.ts @@ -8,6 +8,16 @@ import { CursorPage, type CursorPageParams } from '../../../pagination'; export class Checkpoints extends APIResource { /** * List checkpoints for a fine-tuning job. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const fineTuningJobCheckpoint of client.fineTuning.jobs.checkpoints.list( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * )) { + * // ... + * } + * ``` */ list( fineTuningJobId: string, diff --git a/src/resources/fine-tuning/jobs/jobs.ts b/src/resources/fine-tuning/jobs/jobs.ts index 2198e8174..08616cd4f 100644 --- a/src/resources/fine-tuning/jobs/jobs.ts +++ b/src/resources/fine-tuning/jobs/jobs.ts @@ -3,6 +3,7 @@ import { APIResource } from '../../../resource'; import { isRequestOptions } from '../../../core'; import * as Core from '../../../core'; +import * as MethodsAPI from '../methods'; import * as CheckpointsAPI from './checkpoints'; import { CheckpointListParams, @@ -23,6 +24,14 @@ export class Jobs extends APIResource { * of the fine-tuned models once complete. * * [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + * + * @example + * ```ts + * const fineTuningJob = await client.fineTuning.jobs.create({ + * model: 'gpt-4o-mini', + * training_file: 'file-abc123', + * }); + * ``` */ create(body: JobCreateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/fine_tuning/jobs', { body, ...options }); @@ -32,6 +41,13 @@ export class Jobs extends APIResource { * Get info about a fine-tuning job. * * [Learn more about fine-tuning](https://platform.openai.com/docs/guides/fine-tuning) + * + * @example + * ```ts + * const fineTuningJob = await client.fineTuning.jobs.retrieve( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * ); + * ``` */ retrieve(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.get(`/fine_tuning/jobs/${fineTuningJobId}`, options); @@ -39,6 +55,14 @@ export class Jobs extends APIResource { /** * List your organization's fine-tuning jobs + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const fineTuningJob of client.fineTuning.jobs.list()) { + * // ... + * } + * ``` */ list( query?: JobListParams, @@ -57,6 +81,13 @@ export class Jobs extends APIResource { /** * Immediately cancel a fine-tune job. + * + * @example + * ```ts + * const fineTuningJob = await client.fineTuning.jobs.cancel( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * ); + * ``` */ cancel(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/cancel`, options); @@ -64,6 +95,16 @@ export class Jobs extends APIResource { /** * Get status updates for a fine-tuning job. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const fineTuningJobEvent of client.fineTuning.jobs.listEvents( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * )) { + * // ... + * } + * ``` */ listEvents( fineTuningJobId: string, @@ -87,6 +128,34 @@ export class Jobs extends APIResource { ...options, }); } + + /** + * Pause a fine-tune job. + * + * @example + * ```ts + * const fineTuningJob = await client.fineTuning.jobs.pause( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * ); + * ``` + */ + pause(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/pause`, options); + } + + /** + * Resume a fine-tune job. + * + * @example + * ```ts + * const fineTuningJob = await client.fineTuning.jobs.resume( + * 'ft-AF1WoRqd3aJAHsqc9NY7iL8F', + * ); + * ``` + */ + resume(fineTuningJobId: string, options?: Core.RequestOptions): Core.APIPromise { + return this._client.post(`/fine_tuning/jobs/${fineTuningJobId}/resume`, options); + } } export class FineTuningJobsPage extends CursorPage {} @@ -253,97 +322,24 @@ export namespace FineTuningJob { */ export interface Method { /** - * Configuration for the DPO fine-tuning method. + * The type of method. Is either `supervised`, `dpo`, or `reinforcement`. */ - dpo?: Method.Dpo; + type: 'supervised' | 'dpo' | 'reinforcement'; /** - * Configuration for the supervised fine-tuning method. + * Configuration for the DPO fine-tuning method. */ - supervised?: Method.Supervised; + dpo?: MethodsAPI.DpoMethod; /** - * The type of method. Is either `supervised` or `dpo`. + * Configuration for the reinforcement fine-tuning method. */ - type?: 'supervised' | 'dpo'; - } - - export namespace Method { - /** - * Configuration for the DPO fine-tuning method. - */ - export interface Dpo { - /** - * The hyperparameters used for the fine-tuning job. - */ - hyperparameters?: Dpo.Hyperparameters; - } - - export namespace Dpo { - /** - * The hyperparameters used for the fine-tuning job. - */ - export interface Hyperparameters { - /** - * Number of examples in each batch. A larger batch size means that model - * parameters are updated less frequently, but with lower variance. - */ - batch_size?: 'auto' | number; - - /** - * The beta value for the DPO method. A higher beta value will increase the weight - * of the penalty between the policy and reference model. - */ - beta?: 'auto' | number; - - /** - * Scaling factor for the learning rate. A smaller learning rate may be useful to - * avoid overfitting. - */ - learning_rate_multiplier?: 'auto' | number; - - /** - * The number of epochs to train the model for. An epoch refers to one full cycle - * through the training dataset. - */ - n_epochs?: 'auto' | number; - } - } + reinforcement?: MethodsAPI.ReinforcementMethod; /** * Configuration for the supervised fine-tuning method. */ - export interface Supervised { - /** - * The hyperparameters used for the fine-tuning job. - */ - hyperparameters?: Supervised.Hyperparameters; - } - - export namespace Supervised { - /** - * The hyperparameters used for the fine-tuning job. - */ - export interface Hyperparameters { - /** - * Number of examples in each batch. A larger batch size means that model - * parameters are updated less frequently, but with lower variance. - */ - batch_size?: 'auto' | number; - - /** - * Scaling factor for the learning rate. A smaller learning rate may be useful to - * avoid overfitting. - */ - learning_rate_multiplier?: 'auto' | number; - - /** - * The number of epochs to train the model for. An epoch refers to one full cycle - * through the training dataset. - */ - n_epochs?: 'auto' | number; - } - } + supervised?: MethodsAPI.SupervisedMethod; } } @@ -597,97 +593,24 @@ export namespace JobCreateParams { */ export interface Method { /** - * Configuration for the DPO fine-tuning method. - */ - dpo?: Method.Dpo; - - /** - * Configuration for the supervised fine-tuning method. + * The type of method. Is either `supervised`, `dpo`, or `reinforcement`. */ - supervised?: Method.Supervised; + type: 'supervised' | 'dpo' | 'reinforcement'; /** - * The type of method. Is either `supervised` or `dpo`. + * Configuration for the DPO fine-tuning method. */ - type?: 'supervised' | 'dpo'; - } + dpo?: MethodsAPI.DpoMethod; - export namespace Method { /** - * Configuration for the DPO fine-tuning method. + * Configuration for the reinforcement fine-tuning method. */ - export interface Dpo { - /** - * The hyperparameters used for the fine-tuning job. - */ - hyperparameters?: Dpo.Hyperparameters; - } - - export namespace Dpo { - /** - * The hyperparameters used for the fine-tuning job. - */ - export interface Hyperparameters { - /** - * Number of examples in each batch. A larger batch size means that model - * parameters are updated less frequently, but with lower variance. - */ - batch_size?: 'auto' | number; - - /** - * The beta value for the DPO method. A higher beta value will increase the weight - * of the penalty between the policy and reference model. - */ - beta?: 'auto' | number; - - /** - * Scaling factor for the learning rate. A smaller learning rate may be useful to - * avoid overfitting. - */ - learning_rate_multiplier?: 'auto' | number; - - /** - * The number of epochs to train the model for. An epoch refers to one full cycle - * through the training dataset. - */ - n_epochs?: 'auto' | number; - } - } + reinforcement?: MethodsAPI.ReinforcementMethod; /** * Configuration for the supervised fine-tuning method. */ - export interface Supervised { - /** - * The hyperparameters used for the fine-tuning job. - */ - hyperparameters?: Supervised.Hyperparameters; - } - - export namespace Supervised { - /** - * The hyperparameters used for the fine-tuning job. - */ - export interface Hyperparameters { - /** - * Number of examples in each batch. A larger batch size means that model - * parameters are updated less frequently, but with lower variance. - */ - batch_size?: 'auto' | number; - - /** - * Scaling factor for the learning rate. A smaller learning rate may be useful to - * avoid overfitting. - */ - learning_rate_multiplier?: 'auto' | number; - - /** - * The number of epochs to train the model for. An epoch refers to one full cycle - * through the training dataset. - */ - n_epochs?: 'auto' | number; - } - } + supervised?: MethodsAPI.SupervisedMethod; } } diff --git a/src/resources/fine-tuning/methods.ts b/src/resources/fine-tuning/methods.ts new file mode 100644 index 000000000..aa459c74c --- /dev/null +++ b/src/resources/fine-tuning/methods.ts @@ -0,0 +1,152 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../../resource'; +import * as GraderModelsAPI from '../graders/grader-models'; + +export class Methods extends APIResource {} + +/** + * The hyperparameters used for the DPO fine-tuning job. + */ +export interface DpoHyperparameters { + /** + * Number of examples in each batch. A larger batch size means that model + * parameters are updated less frequently, but with lower variance. + */ + batch_size?: 'auto' | number; + + /** + * The beta value for the DPO method. A higher beta value will increase the weight + * of the penalty between the policy and reference model. + */ + beta?: 'auto' | number; + + /** + * Scaling factor for the learning rate. A smaller learning rate may be useful to + * avoid overfitting. + */ + learning_rate_multiplier?: 'auto' | number; + + /** + * The number of epochs to train the model for. An epoch refers to one full cycle + * through the training dataset. + */ + n_epochs?: 'auto' | number; +} + +/** + * Configuration for the DPO fine-tuning method. + */ +export interface DpoMethod { + /** + * The hyperparameters used for the DPO fine-tuning job. + */ + hyperparameters?: DpoHyperparameters; +} + +/** + * The hyperparameters used for the reinforcement fine-tuning job. + */ +export interface ReinforcementHyperparameters { + /** + * Number of examples in each batch. A larger batch size means that model + * parameters are updated less frequently, but with lower variance. + */ + batch_size?: 'auto' | number; + + /** + * Multiplier on amount of compute used for exploring search space during training. + */ + compute_multiplier?: 'auto' | number; + + /** + * The number of training steps between evaluation runs. + */ + eval_interval?: 'auto' | number; + + /** + * Number of evaluation samples to generate per training step. + */ + eval_samples?: 'auto' | number; + + /** + * Scaling factor for the learning rate. A smaller learning rate may be useful to + * avoid overfitting. + */ + learning_rate_multiplier?: 'auto' | number; + + /** + * The number of epochs to train the model for. An epoch refers to one full cycle + * through the training dataset. + */ + n_epochs?: 'auto' | number; + + /** + * Level of reasoning effort. + */ + reasoning_effort?: 'default' | 'low' | 'medium' | 'high'; +} + +/** + * Configuration for the reinforcement fine-tuning method. + */ +export interface ReinforcementMethod { + /** + * The grader used for the fine-tuning job. + */ + grader: + | GraderModelsAPI.StringCheckGrader + | GraderModelsAPI.TextSimilarityGrader + | GraderModelsAPI.PythonGrader + | GraderModelsAPI.ScoreModelGrader + | GraderModelsAPI.MultiGrader; + + /** + * The hyperparameters used for the reinforcement fine-tuning job. + */ + hyperparameters?: ReinforcementHyperparameters; +} + +/** + * The hyperparameters used for the fine-tuning job. + */ +export interface SupervisedHyperparameters { + /** + * Number of examples in each batch. A larger batch size means that model + * parameters are updated less frequently, but with lower variance. + */ + batch_size?: 'auto' | number; + + /** + * Scaling factor for the learning rate. A smaller learning rate may be useful to + * avoid overfitting. + */ + learning_rate_multiplier?: 'auto' | number; + + /** + * The number of epochs to train the model for. An epoch refers to one full cycle + * through the training dataset. + */ + n_epochs?: 'auto' | number; +} + +/** + * Configuration for the supervised fine-tuning method. + */ +export interface SupervisedMethod { + /** + * The hyperparameters used for the fine-tuning job. + */ + hyperparameters?: SupervisedHyperparameters; +} + +export declare namespace Methods { + export { + type DpoHyperparameters as DpoHyperparameters, + type DpoMethod as DpoMethod, + type ReinforcementHyperparameters as ReinforcementHyperparameters, + type ReinforcementMethod as ReinforcementMethod, + type SupervisedHyperparameters as SupervisedHyperparameters, + type SupervisedMethod as SupervisedMethod, + }; +} diff --git a/src/resources/graders.ts b/src/resources/graders.ts new file mode 100644 index 000000000..2ea9aa959 --- /dev/null +++ b/src/resources/graders.ts @@ -0,0 +1,3 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +export * from './graders/index'; diff --git a/src/resources/graders/grader-models.ts b/src/resources/graders/grader-models.ts new file mode 100644 index 000000000..9ee08f75f --- /dev/null +++ b/src/resources/graders/grader-models.ts @@ -0,0 +1,296 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../../resource'; +import * as ResponsesAPI from '../responses/responses'; + +export class GraderModels extends APIResource {} + +/** + * A LabelModelGrader object which uses a model to assign labels to each item in + * the evaluation. + */ +export interface LabelModelGrader { + input: Array; + + /** + * The labels to assign to each item in the evaluation. + */ + labels: Array; + + /** + * The model to use for the evaluation. Must support structured outputs. + */ + model: string; + + /** + * The name of the grader. + */ + name: string; + + /** + * The labels that indicate a passing result. Must be a subset of labels. + */ + passing_labels: Array; + + /** + * The object type, which is always `label_model`. + */ + type: 'label_model'; +} + +export namespace LabelModelGrader { + /** + * A message input to the model with a role indicating instruction following + * hierarchy. Instructions given with the `developer` or `system` role take + * precedence over instructions given with the `user` role. Messages with the + * `assistant` role are presumed to have been generated by the model in previous + * interactions. + */ + export interface Input { + /** + * Text inputs to the model - can contain template strings. + */ + content: string | ResponsesAPI.ResponseInputText | Input.OutputText; + + /** + * The role of the message input. One of `user`, `assistant`, `system`, or + * `developer`. + */ + role: 'user' | 'assistant' | 'system' | 'developer'; + + /** + * The type of the message input. Always `message`. + */ + type?: 'message'; + } + + export namespace Input { + /** + * A text output from the model. + */ + export interface OutputText { + /** + * The text output from the model. + */ + text: string; + + /** + * The type of the output text. Always `output_text`. + */ + type: 'output_text'; + } + } +} + +/** + * A MultiGrader object combines the output of multiple graders to produce a single + * score. + */ +export interface MultiGrader { + /** + * A formula to calculate the output based on grader results. + */ + calculate_output: string; + + graders: Record< + string, + StringCheckGrader | TextSimilarityGrader | PythonGrader | ScoreModelGrader | LabelModelGrader + >; + + /** + * The name of the grader. + */ + name: string; + + /** + * The type of grader. + */ + type: 'multi'; +} + +/** + * A PythonGrader object that runs a python script on the input. + */ +export interface PythonGrader { + /** + * The name of the grader. + */ + name: string; + + /** + * The source code of the python script. + */ + source: string; + + /** + * The object type, which is always `python`. + */ + type: 'python'; + + /** + * The image tag to use for the python script. + */ + image_tag?: string; +} + +/** + * A ScoreModelGrader object that uses a model to assign a score to the input. + */ +export interface ScoreModelGrader { + /** + * The input text. This may include template strings. + */ + input: Array; + + /** + * The model to use for the evaluation. + */ + model: string; + + /** + * The name of the grader. + */ + name: string; + + /** + * The object type, which is always `score_model`. + */ + type: 'score_model'; + + /** + * The range of the score. Defaults to `[0, 1]`. + */ + range?: Array; + + /** + * The sampling parameters for the model. + */ + sampling_params?: unknown; +} + +export namespace ScoreModelGrader { + /** + * A message input to the model with a role indicating instruction following + * hierarchy. Instructions given with the `developer` or `system` role take + * precedence over instructions given with the `user` role. Messages with the + * `assistant` role are presumed to have been generated by the model in previous + * interactions. + */ + export interface Input { + /** + * Text inputs to the model - can contain template strings. + */ + content: string | ResponsesAPI.ResponseInputText | Input.OutputText; + + /** + * The role of the message input. One of `user`, `assistant`, `system`, or + * `developer`. + */ + role: 'user' | 'assistant' | 'system' | 'developer'; + + /** + * The type of the message input. Always `message`. + */ + type?: 'message'; + } + + export namespace Input { + /** + * A text output from the model. + */ + export interface OutputText { + /** + * The text output from the model. + */ + text: string; + + /** + * The type of the output text. Always `output_text`. + */ + type: 'output_text'; + } + } +} + +/** + * A StringCheckGrader object that performs a string comparison between input and + * reference using a specified operation. + */ +export interface StringCheckGrader { + /** + * The input text. This may include template strings. + */ + input: string; + + /** + * The name of the grader. + */ + name: string; + + /** + * The string check operation to perform. One of `eq`, `ne`, `like`, or `ilike`. + */ + operation: 'eq' | 'ne' | 'like' | 'ilike'; + + /** + * The reference text. This may include template strings. + */ + reference: string; + + /** + * The object type, which is always `string_check`. + */ + type: 'string_check'; +} + +/** + * A TextSimilarityGrader object which grades text based on similarity metrics. + */ +export interface TextSimilarityGrader { + /** + * The evaluation metric to use. One of `fuzzy_match`, `bleu`, `gleu`, `meteor`, + * `rouge_1`, `rouge_2`, `rouge_3`, `rouge_4`, `rouge_5`, or `rouge_l`. + */ + evaluation_metric: + | 'fuzzy_match' + | 'bleu' + | 'gleu' + | 'meteor' + | 'rouge_1' + | 'rouge_2' + | 'rouge_3' + | 'rouge_4' + | 'rouge_5' + | 'rouge_l'; + + /** + * The text being graded. + */ + input: string; + + /** + * The name of the grader. + */ + name: string; + + /** + * The text being graded against. + */ + reference: string; + + /** + * The type of grader. + */ + type: 'text_similarity'; +} + +export declare namespace GraderModels { + export { + type LabelModelGrader as LabelModelGrader, + type MultiGrader as MultiGrader, + type PythonGrader as PythonGrader, + type ScoreModelGrader as ScoreModelGrader, + type StringCheckGrader as StringCheckGrader, + type TextSimilarityGrader as TextSimilarityGrader, + }; +} diff --git a/src/resources/graders/graders.ts b/src/resources/graders/graders.ts new file mode 100644 index 000000000..de3297450 --- /dev/null +++ b/src/resources/graders/graders.ts @@ -0,0 +1,31 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import { APIResource } from '../../resource'; +import * as GraderModelsAPI from './grader-models'; +import { + GraderModels, + LabelModelGrader, + MultiGrader, + PythonGrader, + ScoreModelGrader, + StringCheckGrader, + TextSimilarityGrader, +} from './grader-models'; + +export class Graders extends APIResource { + graderModels: GraderModelsAPI.GraderModels = new GraderModelsAPI.GraderModels(this._client); +} + +Graders.GraderModels = GraderModels; + +export declare namespace Graders { + export { + GraderModels as GraderModels, + type LabelModelGrader as LabelModelGrader, + type MultiGrader as MultiGrader, + type PythonGrader as PythonGrader, + type ScoreModelGrader as ScoreModelGrader, + type StringCheckGrader as StringCheckGrader, + type TextSimilarityGrader as TextSimilarityGrader, + }; +} diff --git a/src/resources/graders/index.ts b/src/resources/graders/index.ts new file mode 100644 index 000000000..82d557a6a --- /dev/null +++ b/src/resources/graders/index.ts @@ -0,0 +1,12 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +export { + GraderModels, + type LabelModelGrader, + type MultiGrader, + type PythonGrader, + type ScoreModelGrader, + type StringCheckGrader, + type TextSimilarityGrader, +} from './grader-models'; +export { Graders } from './graders'; diff --git a/src/resources/images.ts b/src/resources/images.ts index 32f1e123c..c6b14833a 100644 --- a/src/resources/images.ts +++ b/src/resources/images.ts @@ -6,6 +6,13 @@ import * as Core from '../core'; export class Images extends APIResource { /** * Creates a variation of a given image. This endpoint only supports `dall-e-2`. + * + * @example + * ```ts + * const imagesResponse = await client.images.createVariation({ + * image: fs.createReadStream('otter.png'), + * }); + * ``` */ createVariation( body: ImageCreateVariationParams, @@ -17,6 +24,14 @@ export class Images extends APIResource { /** * Creates an edited or extended image given one or more source images and a * prompt. This endpoint only supports `gpt-image-1` and `dall-e-2`. + * + * @example + * ```ts + * const imagesResponse = await client.images.edit({ + * image: fs.createReadStream('path/to/file'), + * prompt: 'A cute baby sea otter wearing a beret', + * }); + * ``` */ edit(body: ImageEditParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/images/edits', Core.multipartFormRequestOptions({ body, ...options })); @@ -25,6 +40,13 @@ export class Images extends APIResource { /** * Creates an image given a prompt. * [Learn more](https://platform.openai.com/docs/guides/images). + * + * @example + * ```ts + * const imagesResponse = await client.images.generate({ + * prompt: 'A cute baby sea otter', + * }); + * ``` */ generate(body: ImageGenerateParams, options?: Core.RequestOptions): Core.APIPromise { return this._client.post('/images/generations', { body, ...options }); diff --git a/src/resources/index.ts b/src/resources/index.ts index 0d8ec9220..9d827615c 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -33,10 +33,7 @@ export { EvalListResponsesPage, Evals, type EvalCustomDataSourceConfig, - type EvalLabelModelGrader, type EvalStoredCompletionsDataSourceConfig, - type EvalStringCheckGrader, - type EvalTextSimilarityGrader, type EvalCreateResponse, type EvalRetrieveResponse, type EvalUpdateResponse, @@ -57,6 +54,7 @@ export { type FileListParams, } from './files'; export { FineTuning } from './fine-tuning/fine-tuning'; +export { Graders } from './graders/graders'; export { Images, type Image, diff --git a/src/resources/responses/input-items.ts b/src/resources/responses/input-items.ts index c88bb441d..74707f184 100644 --- a/src/resources/responses/input-items.ts +++ b/src/resources/responses/input-items.ts @@ -10,6 +10,16 @@ import { type CursorPageParams } from '../../pagination'; export class InputItems extends APIResource { /** * Returns a list of input items for a given response. + * + * @example + * ```ts + * // Automatically fetches more pages as needed. + * for await (const responseItem of client.responses.inputItems.list( + * 'response_id', + * )) { + * // ... + * } + * ``` */ list( responseId: string, diff --git a/src/resources/responses/responses.ts b/src/resources/responses/responses.ts index 0a6e3666d..1440e865e 100644 --- a/src/resources/responses/responses.ts +++ b/src/resources/responses/responses.ts @@ -61,6 +61,14 @@ export class Responses extends APIResource { * [web search](https://platform.openai.com/docs/guides/tools-web-search) or * [file search](https://platform.openai.com/docs/guides/tools-file-search) to use * your own data as input for the model's response. + * + * @example + * ```ts + * const response = await client.responses.create({ + * input: 'string', + * model: 'gpt-4o', + * }); + * ``` */ create(body: ResponseCreateParamsNonStreaming, options?: Core.RequestOptions): APIPromise; create( @@ -90,6 +98,13 @@ export class Responses extends APIResource { /** * Retrieves a model response with the given ID. + * + * @example + * ```ts + * const response = await client.responses.retrieve( + * 'resp_677efb5139a88190b512bc3fef8e535d', + * ); + * ``` */ retrieve( responseId: string, @@ -110,6 +125,13 @@ export class Responses extends APIResource { /** * Deletes a model response with the given ID. + * + * @example + * ```ts + * await client.responses.del( + * 'resp_677efb5139a88190b512bc3fef8e535d', + * ); + * ``` */ del(responseId: string, options?: Core.RequestOptions): Core.APIPromise { return this._client.delete(`/responses/${responseId}`, { diff --git a/src/version.ts b/src/version.ts index 97cbc5900..f64cc03ff 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const VERSION = '4.97.0'; // x-release-please-version +export const VERSION = '4.98.0'; // x-release-please-version diff --git a/tests/api-resources/fine-tuning/alpha/graders.test.ts b/tests/api-resources/fine-tuning/alpha/graders.test.ts new file mode 100644 index 000000000..8e47a4c42 --- /dev/null +++ b/tests/api-resources/fine-tuning/alpha/graders.test.ts @@ -0,0 +1,53 @@ +// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +import OpenAI from 'openai'; +import { Response } from 'node-fetch'; + +const client = new OpenAI({ + apiKey: 'My API Key', + baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010', +}); + +describe('resource graders', () => { + test('run: only required params', async () => { + const responsePromise = client.fineTuning.alpha.graders.run({ + grader: { input: 'input', name: 'name', operation: 'eq', reference: 'reference', type: 'string_check' }, + model_sample: 'model_sample', + reference_answer: 'string', + }); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('run: required and optional params', async () => { + const response = await client.fineTuning.alpha.graders.run({ + grader: { input: 'input', name: 'name', operation: 'eq', reference: 'reference', type: 'string_check' }, + model_sample: 'model_sample', + reference_answer: 'string', + }); + }); + + test('validate: only required params', async () => { + const responsePromise = client.fineTuning.alpha.graders.validate({ + grader: { input: 'input', name: 'name', operation: 'eq', reference: 'reference', type: 'string_check' }, + }); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('validate: required and optional params', async () => { + const response = await client.fineTuning.alpha.graders.validate({ + grader: { input: 'input', name: 'name', operation: 'eq', reference: 'reference', type: 'string_check' }, + }); + }); +}); diff --git a/tests/api-resources/fine-tuning/jobs/jobs.test.ts b/tests/api-resources/fine-tuning/jobs/jobs.test.ts index 4de83a8b7..fe8c9efee 100644 --- a/tests/api-resources/fine-tuning/jobs/jobs.test.ts +++ b/tests/api-resources/fine-tuning/jobs/jobs.test.ts @@ -35,6 +35,7 @@ describe('resource jobs', () => { }, ], method: { + type: 'supervised', dpo: { hyperparameters: { batch_size: 'auto', @@ -43,10 +44,27 @@ describe('resource jobs', () => { n_epochs: 'auto', }, }, + reinforcement: { + grader: { + input: 'input', + name: 'name', + operation: 'eq', + reference: 'reference', + type: 'string_check', + }, + hyperparameters: { + batch_size: 'auto', + compute_multiplier: 'auto', + eval_interval: 'auto', + eval_samples: 'auto', + learning_rate_multiplier: 'auto', + n_epochs: 'auto', + reasoning_effort: 'default', + }, + }, supervised: { hyperparameters: { batch_size: 'auto', learning_rate_multiplier: 'auto', n_epochs: 'auto' }, }, - type: 'supervised', }, seed: 42, suffix: 'x', @@ -143,4 +161,40 @@ describe('resource jobs', () => { ), ).rejects.toThrow(OpenAI.NotFoundError); }); + + test('pause', async () => { + const responsePromise = client.fineTuning.jobs.pause('ft-AF1WoRqd3aJAHsqc9NY7iL8F'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('pause: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect( + client.fineTuning.jobs.pause('ft-AF1WoRqd3aJAHsqc9NY7iL8F', { path: '/_stainless_unknown_path' }), + ).rejects.toThrow(OpenAI.NotFoundError); + }); + + test('resume', async () => { + const responsePromise = client.fineTuning.jobs.resume('ft-AF1WoRqd3aJAHsqc9NY7iL8F'); + const rawResponse = await responsePromise.asResponse(); + expect(rawResponse).toBeInstanceOf(Response); + const response = await responsePromise; + expect(response).not.toBeInstanceOf(Response); + const dataAndResponse = await responsePromise.withResponse(); + expect(dataAndResponse.data).toBe(response); + expect(dataAndResponse.response).toBe(rawResponse); + }); + + test('resume: request options instead of params are passed correctly', async () => { + // ensure the request options are being passed correctly by passing an invalid HTTP method in order to cause an error + await expect( + client.fineTuning.jobs.resume('ft-AF1WoRqd3aJAHsqc9NY7iL8F', { path: '/_stainless_unknown_path' }), + ).rejects.toThrow(OpenAI.NotFoundError); + }); });