feat: Add batch evals api

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.

[Luca Forstner (lforst)]

Batch eval

// openai-batch-eval.tsimport{BatchScorer,BatchTask,defineDurableEval,typeDurableEvalStore,}from"braintrust";importOpenAI,{toFile}from"openai";constopenai=newOpenAI();asyncfunctionsubmitOpenAIBatch(requests: Array<{id: string;body: unknown}>,context: {runId: string;batchId: string},){constjsonl=requests.map(({ id, body })=>JSON.stringify({custom_id: id,method: "POST",url: "/v1/chat/completions",
body,}),).join("\n");constfile=awaitopenai.files.create({file: awaittoFile(Buffer.from(jsonl),`${context.batchId}.jsonl`),purpose: "batch",});constbatch=awaitopenai.batches.create({input_file_id: file.id,endpoint: "/v1/chat/completions",completion_window: "24h",metadata: {braintrust_run_id: context.runId,braintrust_batch_id: context.batchId,},});return{batchId: batch.id};}asyncfunctioncollectOpenAIBatch(submission: {batchId: string}){constbatch=awaitopenai.batches.retrieve(submission.batchId);if(!batch.output_file_id){thrownewError(`OpenAI batch ${batch.id} has no output file`);}constresponse=awaitopenai.files.content(batch.output_file_id);constjsonl=awaitresponse.text();returnjsonl.split("\n").filter(Boolean).map((line)=>JSON.parse(line));}exportfunctionmakeOpenAIBatchEval(store: DurableEvalStore,completion: Completion,){returndefineDurableEval("OpenAI batch demo",{
store,data: [{id: "france",input: "What is the capital of France?",expected: "Paris",},{id: "japan",input: "What is the capital of Japan?",expected: "Tokyo",},],task: newBatchTask<string,string,string,void,Record<string,never>,Submission>({batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "user",content: item.input}],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>({id: row.custom_id,output: rowToOutputTooLazyToImplement(row),}));},}),scores: [newBatchScorer<string,string,string,void,Submission>({name: "openai_judge",batchSize: 100,submit(items,context){returnsubmitOpenAIBatch(items.map((item)=>({id: item.id,body: {model: "gpt-4o-mini",messages: [{role: "system",content:
"Return exactly 1 if the answer is correct, otherwise return exactly 0.",},{role: "user",content: JSON.stringify({answer: item.output,expected: item.expected,}),},],},})),context,);},
completion,asynccollect(submission){return(awaitcollectOpenAIBatch(submission)).map((row)=>{return{id: row.custom_id,score: Number(rowToOutputTooLazyToImplement(row)),};});},}),],});}

Polling example

// openai-poll.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "poll",asyncpoll({ batchId }){constbatch=awaitopenai.batches.retrieve(batchId);if(batch.status==="completed"){return{status: "complete"};}if(["failed","expired","cancelled"].includes(batch.status)){return{status: "failed",error: newError(`OpenAI batch ${batchId}${batch.status}`),};}return{status: "pending"};},},);letresult=awaitdurableEval.start();construnId=status.runId;while(result.status!=="completed"){result=awaitdurableEval.poll({ runId });awaitnewPromise((resolve)=>setTimeout(resolve,10_000));}

Webhook example

// openai-webhook-start.tsimport{DurableEvalRedisStore}from"braintrust";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();try{constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);console.log(JSON.stringify(awaitdurableEval.start(),null,2));}finally{awaitredis.quit();}

Handle batch.completed:

// openai-webhook-server.tsimport{DurableEvalRedisStore}from"braintrust";importexpressfrom"express";importOpenAIfrom"openai";import{createClient}from"redis";import{makeOpenAIBatchEval}from"./openai-batch-eval";constopenai=newOpenAI();constredis=createClient({url: process.env.REDIS_URL});awaitredis.connect();constdurableEval=makeOpenAIBatchEval(newDurableEvalRedisStore({client: redis}),{mode: "webhook",externalId: ({ batchId })=>batchId,},);constapp=express();// Signature verification requires the unparsed request body.app.use(express.text({type: "application/json"}));app.post("/openai-webhook",async(request,response)=>{constevent=awaitopenai.webhooks.unwrap(request.body,request.headers,process.env.OPENAI_WEBHOOK_SECRET!,);if(event.type==="batch.completed"){constbatch=awaitopenai.batches.retrieve(event.data.id);construnId=batch.metadata?.braintrust_run_id;if(!runId){thrownewError(`Batch ${batch.id} has no Braintrust run ID`);}// In production, enqueue this work and return 200 immediately.constresult=awaitdurableEval.processBatchResult({
runId,externalId: batch.id,});console.log(result);}response.sendStatus(200);});app.listen(3000);

[Abhijeet Prasad (AbhiPrasad)]

I think this is a good starting point! We may need to export more types, but we can do that at a later point.