Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); feat(prompts): update prompts by viadezo1er · Pull Request #310 · braintrustdata/bt · GitHub
Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,7 +149,7 @@ Remove-Item -Recurse -Force (Join-Path $env:APPDATA "bt") -ErrorAction SilentlyC
| `bt view` | View logs, traces, and spans |
| `bt projects` | Manage projects (list, create, view, delete) |
| `bt datasets` | Manage remote datasets (list, create, update, view, delete) |
| `bt prompts` | Manage prompts (list, view, delete) |
| `bt prompts` | Manage prompts (list, create, view, update, delete) |
| `bt functions` | Manage functions (list, view, invoke, update, push, pull, delete) |
| `bt tools` | Manage tools (list, view, invoke, update, delete) |
| `bt scorers` | Manage scorers (list, create, view, invoke, update, delete) |
Expand DownExpand Up@@ -184,6 +184,7 @@ bt scorers update helpfulness --messages @messages.json
bt scorers update helpfulness --new-slug answer-helpfulness
bt functions update my-function --name "Updated function" --description "Updated"
bt tools update my-tool --prompt @prompt.txt --new-slug lookup-order
bt prompts update my-prompt --messages @messages.json
```

The API replaces `prompt_data` rather than merging it, so `bt` reads the current definition and sends a materialized replacement with your changes. A concurrent edit can therefore be overwritten.
Expand Down
2 changes: 1 addition & 1 deletion src/functions/create.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -247,7 +247,7 @@ fn resolve_slug(args: &CreateArgs, name: &str) -> Result<String> {
Ok(slug)
}

fn slugify(value: &str) -> String {
pub(crate) fn slugify(value: &str) -> String {
let mut slug = String::new();
let mut pending_separator = false;

Expand Down
2 changes: 1 addition & 1 deletion src/functions/mod.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,7 +22,7 @@ pub(crate) mod prompt_patch;
mod pull;
mod push;
pub(crate) mod report;
mod scorer_config;
pub(crate) mod scorer_config;
mod update;
mod view;

Expand Down
41 changes: 41 additions & 0 deletions src/functions/prompt_patch.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -21,6 +21,20 @@ pub(crate) fn materialize_prompt_data_patch(
.unwrap_or_default();
merge_json_objects(&mut merged, &patch_prompt_data);

// Prompt blocks are a discriminated union. Deep-merging a completion block
// into a chat block (or vice versa) leaves fields from both variants and
// produces invalid prompt data.
if let Some(requested_prompt) = patch_prompt_data.get("prompt") {
let requested_type = requested_prompt.get("type").and_then(Value::as_str);
let existing_type = existing_prompt_data
.and_then(|data| data.get("prompt"))
.and_then(|prompt| prompt.get("type"))
.and_then(Value::as_str);
if requested_type.is_some() && requested_type != existing_type {
merged.insert("prompt".to_string(), requested_prompt.clone());
}
}

if let (Some(requested), Some(parser)) = (
patch_prompt_data.get("parser").and_then(Value::as_object),
merged.get_mut("parser").and_then(Value::as_object_mut),
Expand All@@ -43,6 +57,33 @@ mod tests {

use super::*;

#[test]
fn replaces_prompt_block_when_switching_prompt_kinds() {
let existing = json!({
"prompt": {"type": "completion", "content": "Original"},
"options": {"model": "test-model"}
});
let mut patch = json!({
"prompt_data": {
"prompt": {
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
}
}
});

materialize_prompt_data_patch(&mut patch, Some(&existing));

assert_eq!(
patch["prompt_data"]["prompt"],
json!({
"type": "chat",
"messages": [{"role": "user", "content": "Hello"}]
})
);
assert_eq!(patch["prompt_data"]["options"]["model"], "test-model");
}

#[test]
fn materializes_complete_prompt_data_for_patch() {
let existing = json!({
Expand Down
150 changes: 126 additions & 24 deletions src/functions/update.rs
Original file line numberDiff line numberDiff line change
Expand Up@@ -4,6 +4,7 @@ use dialoguer::Confirm;
use serde_json::{Map, Value};

use crate::{
error::user_error,
ui::{is_interactive, print_command_status, with_spinner, CommandStatus},
utils::{merge_json_objects, read_text_source},
};
Expand DownExpand Up@@ -128,11 +129,21 @@ pub(crate) struct ToolUpdateArgs {
common: CommonUpdateArgs,

/// Replace a completion prompt from inline text, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "messages")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "messages",
allow_hyphen_values = true
)]
prompt: Option<String>,

/// Replace chat messages from inline JSON/YAML, @PATH, or stdin (-).
#[arg(long, value_name = "SOURCE", conflicts_with = "prompt")]
#[arg(
long,
value_name = "SOURCE",
conflicts_with = "prompt",
allow_hyphen_values = true
)]
messages: Option<String>,
}

Expand DownExpand Up@@ -179,7 +190,7 @@ pub(crate) async fn run_scorer(
args: &ScorerUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_scorer_patch_body(args)?;
let body = build_scorer_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -196,7 +207,7 @@ pub(crate) async fn run_tool(
args: &ToolUpdateArgs,
json_output: bool,
) -> Result<()> {
let body = build_tool_patch_body(args)?;
let body = build_tool_patch_body(args).map_err(user_error)?;
run_update(
ctx,
&args.common,
Expand All@@ -214,7 +225,7 @@ pub(crate) async fn run_generic(
json_output: bool,
ft: Option<FunctionTypeFilter>,
) -> Result<()> {
let body = build_common_patch_body(&args.common)?;
let body = build_common_patch_body(&args.common).map_err(user_error)?;
run_update(ctx, &args.common, body, json_output, ft, None).await
}

Expand All@@ -228,7 +239,27 @@ async fn run_update(
) -> Result<()> {
let function = resolve_target_function(ctx, common, ft).await?;
if !function_matches_filter(&function, ft) {
bail!("'{}' is not a {}", function.name, label(ft));
return Err(user_error(anyhow!(
"'{}' is not a {}",
function.name,
label(ft)
)));
}

if let Some(new_slug) = common.new_slug.as_deref() {
if new_slug != function.slug {
if let Some(conflict) =
api::get_function_by_slug(&ctx.client, &ctx.project.id, new_slug, None).await?
{
if conflict.id != function.id {
return Err(user_error(anyhow!(
"--new-slug '{new_slug}' is already used by {} '{}'",
conflict.function_type.as_deref().unwrap_or("function"),
conflict.name
)));
}
}
}
}

let implementation = function
Expand All@@ -238,11 +269,12 @@ async fn run_update(
.and_then(Value::as_str)
.unwrap_or("prompt");
if body.get("prompt_data").is_some() && implementation != "prompt" {
bail!(
"prompt configuration cannot update {}-backed function '{}'",
return Err(user_error(anyhow!(
"prompt configuration cannot update {}-backed {} '{}'. Edit its source and run: bt functions push <path> --if-exists replace",
implementation,
label(ft),
function.name
);
)));
}

if let Some(args) = scorer_args {
Expand All@@ -255,17 +287,24 @@ async fn run_update(
),
(Some("classifier"), true, _) | (Some("scorer"), _, true)
) {
bail!("PATCH cannot change between score and classification output");
return Err(user_error(anyhow!(
"cannot change between score and classification output"
)));
}
if args.allow_no_match.is_some() && function_type != Some("classifier") {
bail!("--allow-no-match applies only to classification output");
return Err(user_error(anyhow!(
"--allow-no-match applies only to classification output"
)));
}
if args.pass_threshold.is_some() && function_type == Some("classifier") {
bail!("--pass-threshold applies only to score output");
return Err(user_error(anyhow!(
"--pass-threshold applies only to score output"
)));
}
}

materialize_prompt_data_patch(&mut body, function.prompt_data.as_ref());
materialize_metadata_patch(&mut body, function.metadata.as_ref());

if !common.yes && is_interactive() {
let confirm = Confirm::new()
Expand DownExpand Up@@ -423,14 +462,14 @@ fn merge_common_fields(
args: &CommonUpdateArgs,
include_metadata: bool,
) -> Result<()> {
for (key, value) in [
("name", args.name.as_deref()),
("slug", args.new_slug.as_deref()),
("description", args.description.as_deref()),
for (key, flag, value) in [
("name", "--name", args.name.as_deref()),
("slug", "--new-slug", args.new_slug.as_deref()),
("description", "--description", args.description.as_deref()),
] {
if let Some(value) = value {
if value.trim().is_empty() && key != "description" {
bail!("--{} cannot be empty", key.replace('_', "-"));
bail!("{flag} cannot be empty");
}
patch.insert(key.to_string(), Value::String(value.to_string()));
}
Expand All@@ -449,6 +488,19 @@ fn merge_common_fields(
Ok(())
}

fn materialize_metadata_patch(patch: &mut Value, existing_metadata: Option<&Value>) {
let Some(patch_metadata) = patch.get("metadata").and_then(Value::as_object).cloned() else {
return;
};

let mut metadata = existing_metadata
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
merge_json_objects(&mut metadata, &patch_metadata);
patch["metadata"] = Value::Object(metadata);
}

fn finish_patch(patch: Map<String, Value>, help_command: &str) -> Result<Value> {
if patch.is_empty() {
bail!("no updates requested. Pass an update flag; see `{help_command}`");
Expand DownExpand Up@@ -485,6 +537,12 @@ mod tests {
args: ScorerUpdateArgs,
}

#[derive(Debug, Parser)]
struct ToolUpdateArgsHarness {
#[command(flatten)]
args: ToolUpdateArgs,
}

fn args(model: Option<&str>, description: Option<&str>) -> ScorerUpdateArgs {
ScorerUpdateArgs {
common: CommonUpdateArgs {
Expand DownExpand Up@@ -668,16 +726,34 @@ mod tests {
);
}

#[test]
fn tool_prompt_accepts_text_beginning_with_a_dash() {
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--prompt",
"- instruction {{value}}",
])
.expect("parse prompt beginning with a dash");

let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(
body["prompt_data"]["prompt"]["content"],
"- instruction {{value}}"
);
}

#[test]
fn tool_messages_accept_yaml() {
let scorer = args(None, None);
let args = ToolUpdateArgs {
common: scorer.common,
prompt: None,
messages: Some("- role: user\n content: Look up {{order_id}}\n".to_string()),
};
let parsed = ToolUpdateArgsHarness::try_parse_from([
"test",
"test-tool",
"--messages",
"- role: user\n content: Look up {{order_id}}\n",
])
.expect("parse inline YAML beginning with a dash");

let body = build_tool_patch_body(&args).expect("patch body");
let body = build_tool_patch_body(&parsed.args).expect("patch body");
assert_eq!(body["prompt_data"]["prompt"]["type"], "chat");
assert_eq!(body["prompt_data"]["prompt"]["messages"][0]["role"], "user");
}
Expand DownExpand Up@@ -762,4 +838,30 @@ mod tests {
serde_json::json!(0)
);
}

#[test]
fn metadata_patch_preserves_existing_fields() {
let mut patch = json!({"metadata": {"owner": "test-team"}});
let existing = json!({"__pass_threshold": 0.6, "phase": "test"});

materialize_metadata_patch(&mut patch, Some(&existing));

assert_eq!(
patch["metadata"],
json!({
"__pass_threshold": 0.6,
"phase": "test",
"owner": "test-team"
})
);
}

#[test]
fn empty_new_slug_names_the_correct_flag() {
let mut args = args(None, None);
args.common.new_slug = Some(String::new());

let error = build_scorer_patch_body(&args).expect_err("empty slug should fail");
assert_eq!(error.to_string(), "--new-slug cannot be empty");
}
}
15 changes: 15 additions & 0 deletions src/prompts/api.rs
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
use anyhow::Result;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use urlencoding::encode;

use crate::http::ApiClient;
Expand DownExpand Up@@ -47,7 +48,21 @@ pub async fn get_prompt_by_slug(
Ok(list.objects.into_iter().next())
}

pub async fn create_prompt(client: &ApiClient, body: &serde_json::Value) -> Result<Prompt> {
client.post("/v1/prompt", body).await
}

pub async fn delete_prompt(client: &ApiClient, prompt_id: &str) -> Result<()> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.delete(&path).await
}

/// Partially update a prompt by id via `PATCH /v1/prompt/{id}`.
///
/// Top-level fields are patched, but object-valued fields such as `prompt_data`
/// are replaced wholesale. Callers updating `prompt_data` must materialize the
/// complete value before sending the request.
pub async fn patch_prompt(client: &ApiClient, prompt_id: &str, body: &Value) -> Result<Prompt> {
let path = format!("/v1/prompt/{}", encode(prompt_id));
client.patch(&path, body).await
}
Loading
Loading