From 04451c74ab37375223987217725e8e0baa96e332 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 23 Mar 2026 15:07:16 -0600 Subject: [PATCH 1/5] downgrade logging for benign instrumentation errors --- .../braintrust/instrumentation/Instrumenter.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/Instrumenter.java b/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/Instrumenter.java index 2bdf9795..e0702559 100644 --- a/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/Instrumenter.java +++ b/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/Instrumenter.java @@ -215,7 +215,18 @@ public void onError( JavaModule module, boolean loaded, Throwable throwable) { - log.error("Error transforming {}", typeName, throwable); + // TypePool resolution failures (NoSuchTypeException) happen when ByteBuddy tries to + // resolve the type hierarchy of a class whose supertype or interface isn't on the + // classpath (e.g. jakarta.servlet.Filter in a non-servlet app). These are harmless — + // the class simply won't be transformed. Downgrade to DEBUG to avoid log noise. + if (throwable instanceof net.bytebuddy.pool.TypePool.Resolution.NoSuchTypeException) { + log.debug( + "Skipping {} — unresolvable type in hierarchy: {}", + typeName, + throwable.getMessage()); + } else { + log.error("Error transforming {}", typeName, throwable); + } } @Override From 84962dca7c2b0b53a3d0c3aa0649ca2c3c013f87 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 23 Mar 2026 14:52:47 -0600 Subject: [PATCH 2/5] support ignoredInstrumentation in muzzle --- .../gradle/muzzle/MuzzleDirective.groovy | 7 +++++++ .../braintrust/gradle/muzzle/MuzzleTask.groovy | 15 +++++++++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy index da122439..a282f9da 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleDirective.groovy @@ -33,6 +33,13 @@ class MuzzleDirective { /** Transitive dependencies to exclude from the test classpath */ List excludedDependencies = [] + /** + * Fully-qualified class names of {@code InstrumentationModule} implementations to skip when + * checking this directive. Use this when a single Gradle project contains modules for multiple + * libraries and a given directive's classpath only satisfies one of them. + */ + List ignoredInstrumentation = [] + void skipVersions(String... versions) { skipVersions.addAll(versions) } diff --git a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy index 63170fac..2d667c30 100644 --- a/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy +++ b/buildSrc/src/main/groovy/dev/braintrust/gradle/muzzle/MuzzleTask.groovy @@ -72,7 +72,7 @@ class MuzzleTask extends DefaultTask { continue } - def result = checkVersion(libraryJars, directive, version, bootstrapCL, instrumentationCL) + def result = checkVersion(libraryJars, directive, version, bootstrapCL, instrumentationCL, directive.ignoredInstrumentation as Set) if (result.passed && directive.assertPass) { logger.lifecycle("[muzzle] ${version} PASS") @@ -187,7 +187,8 @@ class MuzzleTask extends DefaultTask { MuzzleDirective directive, String version, URLClassLoader bootstrapCL, - URLClassLoader instrumentationCL) { + URLClassLoader instrumentationCL, + Set ignoredInstrumentation = []) { def libraryUrls = libraryJars.collect { it.toURI().toURL() } as URL[] @@ -196,7 +197,7 @@ class MuzzleTask extends DefaultTask { def libraryCL = new URLClassLoader(libraryUrls, bootstrapCL) try { - return doCheck(instrumentationCL, libraryCL) + return doCheck(instrumentationCL, libraryCL, ignoredInstrumentation) } finally { libraryCL.close() } @@ -206,7 +207,7 @@ class MuzzleTask extends DefaultTask { * Performs the actual muzzle check: loads modules via ServiceLoader, checks references, * verifies helper injection. */ - private CheckResult doCheck(URLClassLoader instrumentationCL, URLClassLoader libraryCL) { + private CheckResult doCheck(URLClassLoader instrumentationCL, URLClassLoader libraryCL, Set ignoredInstrumentation = []) { def messages = [] // Load classes from the instrumentation classloader @@ -220,6 +221,12 @@ class MuzzleTask extends DefaultTask { anyModule = true def moduleName = module.name() + // 0. Skip modules explicitly excluded for this directive + if (ignoredInstrumentation.contains(module.getClass().getName())) { + logger.info("[muzzle] module '${moduleName}' skipped (listed in ignoredInstrumentation)") + continue + } + // 1. Check classLoaderMatcher def classLoaderMatcher = module.classLoaderMatcher() boolean clMatch = classLoaderMatcher.matches(libraryCL) From 38afbb6f45ea3f659dcad24607f4f5e3dc1f9726 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 23 Mar 2026 16:30:07 -0600 Subject: [PATCH 3/5] muzzle: follow refs from advice + helper classes --- .../instrumentation/muzzle/ReferenceCreator.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/muzzle/ReferenceCreator.java b/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/muzzle/ReferenceCreator.java index af40e10e..33b3a0ff 100644 --- a/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/muzzle/ReferenceCreator.java +++ b/braintrust-java-agent/instrumenter/src/main/java/dev/braintrust/instrumentation/muzzle/ReferenceCreator.java @@ -63,10 +63,20 @@ public static Map createReferencesFrom( Map references = new LinkedHashMap<>(); Queue queue = new ArrayDeque<>(); queue.add(entryPointClassName); + // Always scan all declared helpers unconditionally — don't rely on bytecode reachability. + // A helper may be dispatched to via reflection (e.g. Class.forName + Method.invoke) which + // is invisible to the static scanner, but its references still need to be muzzle-checked. + for (String helperClassName : helperClassNames) { + if (!helperClassName.equals(entryPointClassName)) { + queue.add(helperClassName); + } + } while (!queue.isEmpty()) { String className = queue.remove(); - visitedSources.add(className); + if (!visitedSources.add(className)) { + continue; + } String resourceName = className.replace('.', '/') + ".class"; try (InputStream in = loader.getResourceAsStream(resourceName)) { if (in == null) { From fbc7a5e83c77b9186127491b38f47fcefb88b734 Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Mon, 23 Mar 2026 23:21:55 -0600 Subject: [PATCH 4/5] VCR record --- ...-209da0d9-7370-4740-9503-981c835fda24.json | 1 + ...-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json | 1 + ...s-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt | 24 +++++++++ ...-209da0d9-7370-4740-9503-981c835fda24.json | 53 ++++++++++++++++++ ...-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json | 53 ++++++++++++++++++ ...-fb58e44c-ab06-4f85-a6fd-e0e982440750.json | 54 +++++++++++++++++++ ...-17681b39-34a2-4c2d-a53f-8d060139cf03.json | 39 ++++++++++++++ ...-24991103-5e15-48e3-b494-afa4ed0381ec.json | 39 ++++++++++++++ ...-27ac352d-2a5b-4b1e-bf64-3fcd82933eae.json | 39 ++++++++++++++ ...-2e318708-ffb9-4fcc-a54d-5e516a54f988.json | 39 ++++++++++++++ ...-69d8944a-62b4-4aa4-99eb-07bb05ec277f.json | 39 ++++++++++++++ ...-7e95c59f-6487-4893-8056-4b3e317c30b9.json | 39 ++++++++++++++ ...-5818928c-54ce-46f7-96a2-53893dfefce8.json | 36 +++++++++++++ ...-66ebfe33-532e-4d37-9eef-f52073d3852b.json | 36 +++++++++++++ ...s-d09661be-dc30-4efd-a105-849e05b49ef6.txt | 22 ++++++++ ...-5818928c-54ce-46f7-96a2-53893dfefce8.json | 49 +++++++++++++++++ ...-66ebfe33-532e-4d37-9eef-f52073d3852b.json | 49 +++++++++++++++++ ...-d09661be-dc30-4efd-a105-849e05b49ef6.json | 49 +++++++++++++++++ 18 files changed, 661 insertions(+) create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-17681b39-34a2-4c2d-a53f-8d060139cf03.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-24991103-5e15-48e3-b494-afa4ed0381ec.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-27ac352d-2a5b-4b1e-bf64-3fcd82933eae.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e318708-ffb9-4fcc-a54d-5e516a54f988.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-69d8944a-62b4-4aa4-99eb-07bb05ec277f.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-7e95c59f-6487-4893-8056-4b3e317c30b9.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.txt create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json create mode 100644 test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.json diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json new file mode 100644 index 00000000..0d5a29ad --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json @@ -0,0 +1 @@ +{"model":"claude-3-haiku-20240307","id":"msg_01FcFjeuPmcHRba7NfnotJVq","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json new file mode 100644 index 00000000..0b4aa294 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json @@ -0,0 +1 @@ +{"model":"claude-3-haiku-20240307","id":"msg_01WdD4ocbqtZbkLfywS52wPk","type":"message","role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":21,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":10,"service_tier":"standard","inference_geo":"not_available"}} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt new file mode 100644 index 00000000..87c34cb9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/__files/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt @@ -0,0 +1,24 @@ +event: message_start +data: {"type":"message_start","message":{"model":"claude-3-haiku-20240307","id":"msg_019ae7Li81juf2pUqzeSH5sX","type":"message","role":"assistant","content":[],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":1,"service_tier":"standard","inference_geo":"not_available"}} } + +event: content_block_start +data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""} } + +event: ping +data: {"type": "ping"} + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The"} } + +event: content_block_delta +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" capital of France is Paris."}} + +event: content_block_stop +data: {"type":"content_block_stop","index":0} + +event: message_delta +data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"input_tokens":14,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":10} } + +event: message_stop +data: {"type":"message_stop" } + diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json new file mode 100644 index 00000000..39dde8fe --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-209da0d9-7370-4740-9503-981c835fda24.json @@ -0,0 +1,53 @@ +{ + "id" : "209da0d9-7370-4740-9503-981c835fda24", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"model\":\"claude-3-haiku-20240307\",\"messages\":[{\"content\":[{\"type\":\"text\",\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"system\":\"\",\"max_tokens\":50,\"stream\":false,\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-209da0d9-7370-4740-9503-981c835fda24.json", + "headers" : { + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "x-envoy-upstream-service-time" : "1213", + "Server" : "cloudflare", + "vary" : "Accept-Encoding", + "anthropic-ratelimit-output-tokens-limit" : "1500000", + "anthropic-ratelimit-output-tokens-reset" : "2026-03-24T05:21:45Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-03-24T05:21:44Z", + "anthropic-ratelimit-tokens-remaining" : "9500000", + "set-cookie" : "_cfuvid=SfatMCW.D1A2Fv5E6LO8UorXXYvSYha4BeWxBt4XbK4-1774329704.1853507-1.0.1.1-HRCQ50BqTgyb8EPRktwVvuJvVOq_la2tBxdDCq8NUJw; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com", + "anthropic-ratelimit-requests-limit" : "10000", + "Content-Security-Policy" : "default-src 'none'; frame-ancestors 'none'", + "server-timing" : "x-originResponse;dur=1214", + "anthropic-ratelimit-input-tokens-remaining" : "8000000", + "anthropic-ratelimit-requests-remaining" : "9999", + "Content-Type" : "application/json", + "CF-RAY" : "9e133beb28dfdef0-SEA", + "anthropic-ratelimit-tokens-limit" : "9500000", + "cf-cache-status" : "DYNAMIC", + "request-id" : "req_011CZMLvED4NXVV2Z7jdwvZ4", + "anthropic-ratelimit-tokens-reset" : "2026-03-24T05:21:44Z", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Tue, 24 Mar 2026 05:21:45 GMT", + "X-Robots-Tag" : "none", + "anthropic-ratelimit-requests-reset" : "2026-03-24T05:21:44Z", + "anthropic-ratelimit-input-tokens-limit" : "8000000", + "anthropic-ratelimit-output-tokens-remaining" : "1500000" + } + }, + "uuid" : "209da0d9-7370-4740-9503-981c835fda24", + "persistent" : true, + "insertionIndex" : 26 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json new file mode 100644 index 00000000..16f03bb8 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json @@ -0,0 +1,53 @@ +{ + "id" : "87ebefc6-6fa9-4e6f-aa0b-665450656a60", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"model\":\"claude-3-haiku-20240307\",\"messages\":[{\"content\":[{\"type\":\"text\",\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"system\":\"You are a helpful geography assistant.\",\"max_tokens\":50,\"stream\":false,\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-87ebefc6-6fa9-4e6f-aa0b-665450656a60.json", + "headers" : { + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "x-envoy-upstream-service-time" : "364", + "Server" : "cloudflare", + "vary" : "Accept-Encoding", + "anthropic-ratelimit-output-tokens-limit" : "1500000", + "anthropic-ratelimit-output-tokens-reset" : "2026-03-24T05:21:41Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-03-24T05:21:41Z", + "anthropic-ratelimit-tokens-remaining" : "9500000", + "set-cookie" : "_cfuvid=TtFCiarcTSxH9g2lzU82cXqwoi0H0FqparanWW2fdM0-1774329701.218075-1.0.1.1-DeCwuCusqYdinXYb7ohQsXBNkZtvnl5J.ljwBAn3ztc; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com", + "anthropic-ratelimit-requests-limit" : "10000", + "Content-Security-Policy" : "default-src 'none'; frame-ancestors 'none'", + "server-timing" : "x-originResponse;dur=367", + "anthropic-ratelimit-input-tokens-remaining" : "8000000", + "anthropic-ratelimit-requests-remaining" : "9999", + "Content-Type" : "application/json", + "CF-RAY" : "9e133bd89bcf75a0-SEA", + "anthropic-ratelimit-tokens-limit" : "9500000", + "cf-cache-status" : "DYNAMIC", + "request-id" : "req_011CZMLv1Y4c3K7FLqnTPUXu", + "anthropic-ratelimit-tokens-reset" : "2026-03-24T05:21:41Z", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Tue, 24 Mar 2026 05:21:41 GMT", + "X-Robots-Tag" : "none", + "anthropic-ratelimit-requests-reset" : "2026-03-24T05:21:41Z", + "anthropic-ratelimit-input-tokens-limit" : "8000000", + "anthropic-ratelimit-output-tokens-remaining" : "1500000" + } + }, + "uuid" : "87ebefc6-6fa9-4e6f-aa0b-665450656a60", + "persistent" : true, + "insertionIndex" : 27 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.json b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.json new file mode 100644 index 00000000..75f15af1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/anthropic/mappings/v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.json @@ -0,0 +1,54 @@ +{ + "id" : "fb58e44c-ab06-4f85-a6fd-e0e982440750", + "name" : "v1_messages", + "request" : { + "url" : "/v1/messages", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"model\":\"claude-3-haiku-20240307\",\"messages\":[{\"content\":[{\"type\":\"text\",\"text\":\"What is the capital of France?\"}],\"role\":\"user\"}],\"system\":\"\",\"max_tokens\":50,\"stream\":true,\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "v1_messages-fb58e44c-ab06-4f85-a6fd-e0e982440750.txt", + "headers" : { + "anthropic-organization-id" : "27796668-7351-40ac-acc4-024aee8995a5", + "x-envoy-upstream-service-time" : "365", + "Server" : "cloudflare", + "vary" : "Accept-Encoding", + "anthropic-ratelimit-output-tokens-limit" : "1500000", + "anthropic-ratelimit-output-tokens-reset" : "2026-03-24T05:21:48Z", + "anthropic-ratelimit-input-tokens-reset" : "2026-03-24T05:21:48Z", + "anthropic-ratelimit-tokens-remaining" : "9500000", + "set-cookie" : "_cfuvid=thVxqAn1MAu.ZVF1i3CFr5kHWdGFtaXSox4YChe7eL8-1774329708.2827516-1.0.1.1-C7bikHGed59W3Mc2pozL47k3kYDz6DlwP8NqKf6dvrc; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.anthropic.com", + "anthropic-ratelimit-requests-limit" : "10000", + "Content-Security-Policy" : "default-src 'none'; frame-ancestors 'none'", + "server-timing" : "x-originResponse;dur=367", + "anthropic-ratelimit-input-tokens-remaining" : "8000000", + "anthropic-ratelimit-requests-remaining" : "9999", + "Content-Type" : "text/event-stream; charset=utf-8", + "CF-RAY" : "9e133c04c94175a6-SEA", + "anthropic-ratelimit-tokens-limit" : "9500000", + "cf-cache-status" : "DYNAMIC", + "request-id" : "req_011CZMLvXnjTBmaqAbhtMejy", + "anthropic-ratelimit-tokens-reset" : "2026-03-24T05:21:48Z", + "strict-transport-security" : "max-age=31536000; includeSubDomains; preload", + "Date" : "Tue, 24 Mar 2026 05:21:48 GMT", + "X-Robots-Tag" : "none", + "anthropic-ratelimit-requests-reset" : "2026-03-24T05:21:48Z", + "Cache-Control" : "no-cache", + "anthropic-ratelimit-input-tokens-limit" : "8000000", + "anthropic-ratelimit-output-tokens-remaining" : "1500000" + } + }, + "uuid" : "fb58e44c-ab06-4f85-a6fd-e0e982440750", + "persistent" : true, + "insertionIndex" : 25 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-17681b39-34a2-4c2d-a53f-8d060139cf03.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-17681b39-34a2-4c2d-a53f-8d060139cf03.json new file mode 100644 index 00000000..301672b1 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-17681b39-34a2-4c2d-a53f-8d060139cf03.json @@ -0,0 +1,39 @@ +{ + "id" : "17681b39-34a2-4c2d-a53f-8d060139cf03", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Cv8GCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBLHBQoRCg9icmFpbnRydXN0LWphdmESsQUKEFW2zkIrcBx+dscDdH+Fa8oSCC+kXaauLJAgKhlhbnRocm9waWMubWVzc2FnZXMuY3JlYXRlMAE5gvyIFHevnxhBgYG1cHevnxhKuQEKE2JyYWludHJ1c3QubWV0YWRhdGESoQEKngF7InByb3ZpZGVyIjoiYW50aHJvcGljIiwicmVxdWVzdF9wYXRoIjoidjEvbWVzc2FnZXMiLCJtb2RlbCI6ImNsYXVkZS0zLWhhaWt1LTIwMjQwMzA3IiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHBzOi8vYXBpLmFudGhyb3BpYy5jb20iLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqRAQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhJ3CnV7ImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifV0sInVzYWdlIjp7ImlucHV0X3Rva2VucyI6MTQsIm91dHB1dF90b2tlbnMiOjEwfX1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SlcKFWJyYWludHJ1c3QuaW5wdXRfanNvbhI+CjxbeyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTwoSYnJhaW50cnVzdC5tZXRyaWNzEjkKN3siY29tcGxldGlvbl90b2tlbnMiOjEwLCJwcm9tcHRfdG9rZW5zIjoxNCwidG9rZW5zIjoyNH16AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnYlFRyoAMEoSQ=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f69-402e171318f4f335594f75b6;Parent=337c238af9e66f31;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:45 GMT", + "Via" : "1.1 a642518ef4d5fb78c3190de112922a38.cloudfront.net (CloudFront), 1.1 e82f2bd1d85893fad1abb144337e7368.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f690000000042b5cb643f957cfa", + "x-amzn-RequestId" : "987882eb-c680-46f7-866a-9c86845872ab", + "X-Amz-Cf-Id" : "6S36tVkdtfb3K1DdPcEyiGG6wGbbrXdpnxuusls-qjbmyiRPUa2pCw==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "17681b39-34a2-4c2d-a53f-8d060139cf03", + "persistent" : true, + "insertionIndex" : 140 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-24991103-5e15-48e3-b494-afa4ed0381ec.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-24991103-5e15-48e3-b494-afa4ed0381ec.json new file mode 100644 index 00000000..c846b89b --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-24991103-5e15-48e3-b494-afa4ed0381ec.json @@ -0,0 +1,39 @@ +{ + "id" : "24991103-5e15-48e3-b494-afa4ed0381ec", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CtoGCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBKiBQoRCg9icmFpbnRydXN0LWphdmESjAUKEATaEEpkkUOPCVbisM7OMoUSCPLygdyDa+hcKg9DaGF0IENvbXBsZXRpb24wATmUY/Wod6+fGEEHII0HeK+fGEqvAQoTYnJhaW50cnVzdC5tZXRhZGF0YRKXAQqUAXsicHJvdmlkZXIiOiJvcGVuYWkiLCJyZXF1ZXN0X3BhdGgiOiJ2MS9jaGF0L2NvbXBsZXRpb25zIiwibW9kZWwiOiJncHQtNG8tbWluaSIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwczovL2FwaS5vcGVuYWkuY29tIiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn1KgQEKFmJyYWludHJ1c3Qub3V0cHV0X2pzb24SZwplW3sibWVzc2FnZSI6eyJyb2xlIjoiYXNzaXN0YW50IiwiY29udGVudCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifSwiZmluaXNoX3JlYXNvbiI6InN0b3AifV1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SlcKFWJyYWludHJ1c3QuaW5wdXRfanNvbhI+CjxbeyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjcsInByb21wdF90b2tlbnMiOjE0LCJ0b2tlbnMiOjIxfXoAhQEBAQAA" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnZJEqXoAMEJSw=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f6d-546c30b4535f8f647ee20599;Parent=6e516bc74ebc0706;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:49 GMT", + "Via" : "1.1 940972e9e344075576fe20d5db482122.cloudfront.net (CloudFront), 1.1 ee5f8da78d4211a93c9dba8864a4067e.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f6d000000003858af75a0c5c172", + "x-amzn-RequestId" : "59350dab-07f9-43c2-9861-8f484f796a7e", + "X-Amz-Cf-Id" : "y1QhL0Khct2YXhMA7a6IJJNPfUGOziwGXRjtYAhXKRDAcIgZaS5vsw==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "24991103-5e15-48e3-b494-afa4ed0381ec", + "persistent" : true, + "insertionIndex" : 139 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-27ac352d-2a5b-4b1e-bf64-3fcd82933eae.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-27ac352d-2a5b-4b1e-bf64-3fcd82933eae.json new file mode 100644 index 00000000..b46a148f --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-27ac352d-2a5b-4b1e-bf64-3fcd82933eae.json @@ -0,0 +1,39 @@ +{ + "id" : "27ac352d-2a5b-4b1e-bf64-3fcd82933eae", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CscHCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBKPBgoRCg9icmFpbnRydXN0LWphdmES+QUKELXYeZI8jYGPhxZrlYzhh1gSCAfdDc0+Rx81KhlhbnRocm9waWMubWVzc2FnZXMuY3JlYXRlMAE5vx2SY3avnxhB+NzzhXavnxhKuQEKE2JyYWludHJ1c3QubWV0YWRhdGESoQEKngF7InByb3ZpZGVyIjoiYW50aHJvcGljIiwicmVxdWVzdF9wYXRoIjoidjEvbWVzc2FnZXMiLCJtb2RlbCI6ImNsYXVkZS0zLWhhaWt1LTIwMjQwMzA3IiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHBzOi8vYXBpLmFudGhyb3BpYy5jb20iLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqRAQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhJ3CnV7ImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifV0sInVzYWdlIjp7ImlucHV0X3Rva2VucyI6MjEsIm91dHB1dF90b2tlbnMiOjEwfX1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0Sp4BChVicmFpbnRydXN0LmlucHV0X2pzb24ShAEKgQFbeyJyb2xlIjoic3lzdGVtIiwiY29udGVudCI6IllvdSBhcmUgYSBoZWxwZnVsIGdlb2dyYXBoeSBhc3Npc3RhbnQuIn0seyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTwoSYnJhaW50cnVzdC5tZXRyaWNzEjkKN3siY29tcGxldGlvbl90b2tlbnMiOjEwLCJwcm9tcHRfdG9rZW5zIjoyMSwidG9rZW5zIjozMX16AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnX-GAJIAMECsA=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f65-04d275353acd34cf600ba485;Parent=45747fdbd4d274f5;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:42 GMT", + "Via" : "1.1 59e4792b9d6184bfa491a317b36590d2.cloudfront.net (CloudFront), 1.1 ee5f8da78d4211a93c9dba8864a4067e.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f65000000000e5ed4219c5ab05b", + "x-amzn-RequestId" : "2cad0a30-19d4-414f-98d1-1be66b340f45", + "X-Amz-Cf-Id" : "mJ6OfPXdSubdJ9rfIiHmQVUPcJq-Mn4gKRFnWNc0dfnbJ_J7DdN5GA==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "27ac352d-2a5b-4b1e-bf64-3fcd82933eae", + "persistent" : true, + "insertionIndex" : 142 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e318708-ffb9-4fcc-a54d-5e516a54f988.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e318708-ffb9-4fcc-a54d-5e516a54f988.json new file mode 100644 index 00000000..73bc0889 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-2e318708-ffb9-4fcc-a54d-5e516a54f988.json @@ -0,0 +1,39 @@ +{ + "id" : "2e318708-ffb9-4fcc-a54d-5e516a54f988", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "Cv8GCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBLHBQoRCg9icmFpbnRydXN0LWphdmESsQUKEP3p0oI00VJ2EYajdFMgm+8SCBMvNV1/XoMvKhlhbnRocm9waWMubWVzc2FnZXMuY3JlYXRlMAE5TfwyCHivnxhBV85GRHivnxhKuQEKE2JyYWludHJ1c3QubWV0YWRhdGESoQEKngF7InByb3ZpZGVyIjoiYW50aHJvcGljIiwicmVxdWVzdF9wYXRoIjoidjEvbWVzc2FnZXMiLCJtb2RlbCI6ImNsYXVkZS0zLWhhaWt1LTIwMjQwMzA3IiwicmVxdWVzdF9iYXNlX3VyaSI6Imh0dHBzOi8vYXBpLmFudGhyb3BpYy5jb20iLCJyZXF1ZXN0X21ldGhvZCI6IlBPU1QifUqRAQoWYnJhaW50cnVzdC5vdXRwdXRfanNvbhJ3CnV7ImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifV0sInVzYWdlIjp7ImlucHV0X3Rva2VucyI6MTQsIm91dHB1dF90b2tlbnMiOjEwfX1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SlcKFWJyYWludHJ1c3QuaW5wdXRfanNvbhI+CjxbeyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTwoSYnJhaW50cnVzdC5tZXRyaWNzEjkKN3siY29tcGxldGlvbl90b2tlbnMiOjEwLCJwcm9tcHRfdG9rZW5zIjoxNCwidG9rZW5zIjoyNH16AIUBAQEAAA==" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnZOFIKIAMEjYQ=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f6d-08561cf370ea22d51b127abd;Parent=40f4aa3d99232162;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:50 GMT", + "Via" : "1.1 7fcfc911845f681c235b0b3f32f3e1c6.cloudfront.net (CloudFront), 1.1 65f2e9f7f1475de54aa452d3ceb9bcf6.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f6d000000007459b139a87ad8e4", + "x-amzn-RequestId" : "dbedc5e4-b208-4165-8140-0053be850974", + "X-Amz-Cf-Id" : "LoiTuirlYqghCyprzBx3aDb4xpL46w4FOWyQ2-hIaNQ3t90SYiZsaQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "2e318708-ffb9-4fcc-a54d-5e516a54f988", + "persistent" : true, + "insertionIndex" : 138 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-69d8944a-62b4-4aa4-99eb-07bb05ec277f.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-69d8944a-62b4-4aa4-99eb-07bb05ec277f.json new file mode 100644 index 00000000..b3a62eb9 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-69d8944a-62b4-4aa4-99eb-07bb05ec277f.json @@ -0,0 +1,39 @@ +{ + "id" : "69d8944a-62b4-4aa4-99eb-07bb05ec277f", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CtoGCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBKiBQoRCg9icmFpbnRydXN0LWphdmESjAUKEB1DOpKhcvDKTOkqQr3B2moSCFssjRCyHVYKKg9DaGF0IENvbXBsZXRpb24wATkqhAyodq+fGEHdPir3dq+fGEqvAQoTYnJhaW50cnVzdC5tZXRhZGF0YRKXAQqUAXsicHJvdmlkZXIiOiJvcGVuYWkiLCJyZXF1ZXN0X3BhdGgiOiJ2MS9jaGF0L2NvbXBsZXRpb25zIiwibW9kZWwiOiJncHQtNG8tbWluaSIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwczovL2FwaS5vcGVuYWkuY29tIiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn1KgQEKFmJyYWludHJ1c3Qub3V0cHV0X2pzb24SZwplW3sibWVzc2FnZSI6eyJyb2xlIjoiYXNzaXN0YW50IiwiY29udGVudCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifSwiZmluaXNoX3JlYXNvbiI6InN0b3AifV1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0SlcKFWJyYWludHJ1c3QuaW5wdXRfanNvbhI+CjxbeyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjcsInByb21wdF90b2tlbnMiOjE0LCJ0b2tlbnMiOjIxfXoAhQEBAQAA" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnYRFJjoAMECPg=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f67-167ff53536471bb2269517f9;Parent=0623b959320d0409;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:43 GMT", + "Via" : "1.1 95a087e13956fc03ffaeeaec4faa051a.cloudfront.net (CloudFront), 1.1 ddea1c07643e5e0bfceb34480eebdc52.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f670000000010c7f59853d4ab48", + "x-amzn-RequestId" : "73ad6c14-a78a-44de-a12c-05890aae3dd5", + "X-Amz-Cf-Id" : "Qy9Obbn_6Uavw6pSCAOJAm8MHgGeIiBvAazPkByZLmQxuGCCdiCNKQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "69d8944a-62b4-4aa4-99eb-07bb05ec277f", + "persistent" : true, + "insertionIndex" : 141 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-7e95c59f-6487-4893-8056-4b3e317c30b9.json b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-7e95c59f-6487-4893-8056-4b3e317c30b9.json new file mode 100644 index 00000000..f4b4a566 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/braintrust/mappings/otel_v1_traces-7e95c59f-6487-4893-8056-4b3e317c30b9.json @@ -0,0 +1,39 @@ +{ + "id" : "7e95c59f-6487-4893-8056-4b3e317c30b9", + "name" : "otel_v1_traces", + "request" : { + "url" : "/otel/v1/traces", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/x-protobuf" + } + }, + "bodyPatterns" : [ { + "binaryEqualTo" : "CqIHCrIBCiAKDHNlcnZpY2UubmFtZRIQCg5icmFpbnRydXN0LWFwcAoiCg9zZXJ2aWNlLnZlcnNpb24SDwoNMC4yLjktYzNjODk2MAogChZ0ZWxlbWV0cnkuc2RrLmxhbmd1YWdlEgYKBGphdmEKJQoSdGVsZW1ldHJ5LnNkay5uYW1lEg8KDW9wZW50ZWxlbWV0cnkKIQoVdGVsZW1ldHJ5LnNkay52ZXJzaW9uEggKBjEuNTkuMBLqBQoRCg9icmFpbnRydXN0LWphdmES1AUKEKTX8mC1sGH4RbgG3I9AONwSCFtJC4dRLPg5Kg9DaGF0IENvbXBsZXRpb24wATk2sHS8da+fGEHLh+Isdq+fGEqvAQoTYnJhaW50cnVzdC5tZXRhZGF0YRKXAQqUAXsicHJvdmlkZXIiOiJvcGVuYWkiLCJyZXF1ZXN0X3BhdGgiOiJ2MS9jaGF0L2NvbXBsZXRpb25zIiwibW9kZWwiOiJncHQtNG8tbWluaSIsInJlcXVlc3RfYmFzZV91cmkiOiJodHRwczovL2FwaS5vcGVuYWkuY29tIiwicmVxdWVzdF9tZXRob2QiOiJQT1NUIn1KgQEKFmJyYWludHJ1c3Qub3V0cHV0X2pzb24SZwplW3sibWVzc2FnZSI6eyJyb2xlIjoiYXNzaXN0YW50IiwiY29udGVudCI6IlRoZSBjYXBpdGFsIG9mIEZyYW5jZSBpcyBQYXJpcy4ifSwiZmluaXNoX3JlYXNvbiI6InN0b3AifV1KLgoaYnJhaW50cnVzdC5zcGFuX2F0dHJpYnV0ZXMSEAoOeyJ0eXBlIjoibGxtIn1KMgoRYnJhaW50cnVzdC5wYXJlbnQSHQobcHJvamVjdF9uYW1lOmphdmEtdW5pdC10ZXN0Sp4BChVicmFpbnRydXN0LmlucHV0X2pzb24ShAEKgQFbeyJyb2xlIjoic3lzdGVtIiwiY29udGVudCI6IllvdSBhcmUgYSBoZWxwZnVsIGdlb2dyYXBoeSBhc3Npc3RhbnQuIn0seyJyb2xlIjoidXNlciIsImNvbnRlbnQiOiJXaGF0IGlzIHRoZSBjYXBpdGFsIG9mIEZyYW5jZT8ifV1KTgoSYnJhaW50cnVzdC5tZXRyaWNzEjgKNnsiY29tcGxldGlvbl90b2tlbnMiOjcsInByb21wdF90b2tlbnMiOjI1LCJ0b2tlbnMiOjMyfXoAhQEBAQAA" + } ] + }, + "response" : { + "status" : 200, + "headers" : { + "X-Cache" : "Miss from cloudfront", + "x-amz-apigw-id" : "atnXyGZDIAMEQ-w=", + "vary" : "Origin", + "x-amzn-Remapped-content-length" : "0", + "X-Amz-Cf-Pop" : [ "SEA900-P1", "SEA900-P10" ], + "X-Amzn-Trace-Id" : "Root=1-69c21f64-0df548117e6a6da47e9bc5d8;Parent=0f4ce8a35dacc69c;Sampled=0;Lineage=1:24be3d11:0", + "Date" : "Tue, 24 Mar 2026 05:21:40 GMT", + "Via" : "1.1 b7d7903ada432685f0e90f0ca261d864.cloudfront.net (CloudFront), 1.1 0eb43913f9caf453beb959a8a836a688.cloudfront.net (CloudFront)", + "access-control-expose-headers" : "x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms", + "access-control-allow-credentials" : "true", + "x-bt-internal-trace-id" : "69c21f64000000003972d6fbb7b029a4", + "x-amzn-RequestId" : "2512cf0a-22ba-4564-ad58-79339efcd4ef", + "X-Amz-Cf-Id" : "bT5tzOZgJkn5OQogxhm_1qiZXi6Vh_V--dzJchwejZ6iZJQopwG1KQ==", + "etag" : "W/\"0-2jmj7l5rSw0yVb/vlWAYkK/YBwk\"", + "Content-Type" : "application/x-protobuf" + } + }, + "uuid" : "7e95c59f-6487-4893-8056-4b3e317c30b9", + "persistent" : true, + "insertionIndex" : 143 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json new file mode 100644 index 00000000..c402e888 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DMoUtqAPVwlNRwE6gutecKbtj4vfq", + "object": "chat.completion", + "created": 1774329699, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 25, + "completion_tokens": 7, + "total_tokens": 32, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ca3e7d71bf" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json new file mode 100644 index 00000000..47663f69 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json @@ -0,0 +1,36 @@ +{ + "id": "chatcmpl-DMoUwruWZjMy42fUPqe2tLCjc0yWt", + "object": "chat.completion", + "created": 1774329702, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The capital of France is Paris.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 14, + "completion_tokens": 7, + "total_tokens": 21, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ca3e7d71bf" +} diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.txt b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.txt new file mode 100644 index 00000000..bc7e8b59 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/__files/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.txt @@ -0,0 +1,22 @@ +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"fHJ7uE8u5"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"MqotWQ3X"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":" capital"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"HFz"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":" of"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"Aye4vpPe"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":" France"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"8aOB"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"hjfzEYDh"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":" Paris"},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"q8FNP"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null,"obfuscation":"ibGw2RMdip"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null,"obfuscation":"hLZhB"} + +data: {"id":"chatcmpl-DMoV1a2qyBOgoTIDIoavr1NecqPMo","object":"chat.completion.chunk","created":1774329707,"model":"gpt-4o-mini-2024-07-18","service_tier":"default","system_fingerprint":"fp_ca3e7d71bf","choices":[],"usage":{"prompt_tokens":14,"completion_tokens":7,"total_tokens":21,"prompt_tokens_details":{"cached_tokens":0,"audio_tokens":0},"completion_tokens_details":{"reasoning_tokens":0,"audio_tokens":0,"accepted_prediction_tokens":0,"rejected_prediction_tokens":0}},"obfuscation":"uee7nGl4Php"} + +data: [DONE] + diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json new file mode 100644 index 00000000..3f0ba3ce --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json @@ -0,0 +1,49 @@ +{ + "id" : "5818928c-54ce-46f7-96a2-53893dfefce8", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"You are a helpful geography assistant.\",\"role\":\"system\"},{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":50,\"stream\":false,\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-5818928c-54ce-46f7-96a2-53893dfefce8.json", + "headers" : { + "x-request-id" : "req_b28c516c0dbe469aac265ab6a7ec0da1", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e133bc93cbb5ea8-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999980", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 24 Mar 2026 05:21:39 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=s6qI2cxD1e99uhIrKh7QyPlfV5.E3rF90fGB7NkmCcQ-1774329698.7542517-1.0.1.1-GqrJi25J74vtIlBs.WrhMkdr2kSDX.54NqwK01D6eecKgFMJLbT0np5SJQJj2jzP4U3hxFNpO3ks19alez0Sy4rIjp1yEtF.L62UzMXLXYNuuPCsWihDG0fSfSuSqt1L; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 24 Mar 2026 05:51:39 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "629", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "5818928c-54ce-46f7-96a2-53893dfefce8", + "persistent" : true, + "insertionIndex" : 21 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json new file mode 100644 index 00000000..54d2c266 --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json @@ -0,0 +1,49 @@ +{ + "id" : "66ebfe33-532e-4d37-9eef-f52073d3852b", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":50,\"stream\":false,\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-66ebfe33-532e-4d37-9eef-f52073d3852b.json", + "headers" : { + "x-request-id" : "req_b10b6d1fac124ce0a55865916276b73a", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e133bdfcddeeb7b-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999990", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 24 Mar 2026 05:21:43 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=Zcj_j8fcwOPeG75O3ZaE0pRDKCm8zwbwOQOTH_khXeM-1774329702.3701656-1.0.1.1-C5cWDsZ6vstzylsop_Bg2a4t5Yy8AYRbieir_UTtKNNvO5hiYXpHnqfHuDJo.wZ7xs06dH7t2Kuh.txzLfQklRdC8UHi.1AD7m.e.78fz4mIGFK89je9j7iRIfN.rm5z; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 24 Mar 2026 05:51:43 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "345", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "application/json" + } + }, + "uuid" : "66ebfe33-532e-4d37-9eef-f52073d3852b", + "persistent" : true, + "insertionIndex" : 20 +} \ No newline at end of file diff --git a/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.json b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.json new file mode 100644 index 00000000..d6adbeff --- /dev/null +++ b/test-harness/src/testFixtures/resources/cassettes/openai/mappings/chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.json @@ -0,0 +1,49 @@ +{ + "id" : "d09661be-dc30-4efd-a105-849e05b49ef6", + "name" : "chat_completions", + "request" : { + "url" : "/chat/completions", + "method" : "POST", + "headers" : { + "Content-Type" : { + "equalTo" : "application/json" + } + }, + "bodyPatterns" : [ { + "equalToJson" : "{\"messages\":[{\"content\":\"What is the capital of France?\",\"role\":\"user\"}],\"model\":\"gpt-4o-mini\",\"max_tokens\":50,\"stream\":true,\"stream_options\":{\"include_usage\":true},\"temperature\":0.0}", + "ignoreArrayOrder" : true, + "ignoreExtraElements" : false + } ] + }, + "response" : { + "status" : 200, + "bodyFileName" : "chat_completions-d09661be-dc30-4efd-a105-849e05b49ef6.txt", + "headers" : { + "x-request-id" : "req_4abd26cc51c54e21b18708f53f77a5b2", + "x-ratelimit-limit-tokens" : "150000000", + "openai-organization" : "braintrust-data", + "Server" : "cloudflare", + "CF-Ray" : "9e133bfdbee776ce-SEA", + "X-Content-Type-Options" : "nosniff", + "x-ratelimit-reset-requests" : "2ms", + "x-ratelimit-remaining-tokens" : "149999990", + "x-openai-proxy-wasm" : "v0.1", + "x-ratelimit-remaining-requests" : "29999", + "Date" : "Tue, 24 Mar 2026 05:21:47 GMT", + "x-ratelimit-reset-tokens" : "0s", + "access-control-expose-headers" : "X-Request-ID", + "set-cookie" : "__cf_bm=SGfHBQFcNmTC1_ax21eIKL83VC.ehy0GL31hLWCvmig-1774329707.1518722-1.0.1.1-SpmDrB8QBff5l7YSEszFuw7RIkxXypOvW3265IDBZBgYOM8HpoJlhHxo1pjJVlemZGYJ6Xh1sYQpMRNImO6sbLYddCVcEmRtgyLSj3fkeP5XH9KD___CXyZvVm.kdfru; HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Tue, 24 Mar 2026 05:51:47 GMT", + "Strict-Transport-Security" : "max-age=31536000; includeSubDomains; preload", + "CF-Cache-Status" : "DYNAMIC", + "x-ratelimit-limit-requests" : "30000", + "openai-version" : "2020-10-01", + "openai-processing-ms" : "213", + "alt-svc" : "h3=\":443\"; ma=86400", + "openai-project" : "proj_vsCSXafhhByzWOThMrJcZiw9", + "Content-Type" : "text/event-stream; charset=utf-8" + } + }, + "uuid" : "d09661be-dc30-4efd-a105-849e05b49ef6", + "persistent" : true, + "insertionIndex" : 19 +} \ No newline at end of file From 9c5e9dace61ffa77464cd2986fd12f703eef93ca Mon Sep 17 00:00:00 2001 From: Andrew Kent Date: Fri, 20 Mar 2026 11:29:25 -0600 Subject: [PATCH 5/5] enhance SpringAI instrumentation --- .../springai_1_0_0/build.gradle | 70 ++++ .../v1_0_0/AnthropicBuilderWrapper.java | 141 ++++++++ .../v1_0_0/BraintrustObservationHandler.java | 76 +++++ .../springai/v1_0_0/BraintrustSpringAI.java | 51 +++ .../springai/v1_0_0/OpenAIBuilderWrapper.java | 163 +++++++++ ...pringAIAnthropicInstrumentationModule.java | 61 ++++ .../SpringAIOpenAIInstrumentationModule.java | 62 ++++ .../v1_0_0/BraintrustSpringAITest.java | 314 ++++++++++++++++++ braintrust-sdk/build.gradle | 2 +- examples/build.gradle | 7 +- .../braintrust/examples/SpringAIExample.java | 64 +++- settings.gradle | 1 + test-harness/build.gradle | 2 +- 13 files changed, 1005 insertions(+), 9 deletions(-) create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/build.gradle create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/AnthropicBuilderWrapper.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustObservationHandler.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/OpenAIBuilderWrapper.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java create mode 100644 braintrust-java-agent/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/build.gradle b/braintrust-java-agent/instrumentation/springai_1_0_0/build.gradle new file mode 100644 index 00000000..06fc820f --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/build.gradle @@ -0,0 +1,70 @@ +def springAiVersion = '1.0.0' + +muzzle { + pass { + group = 'org.springframework.ai' + module = 'spring-ai-openai' + versions = "[${springAiVersion},)" + ignoredInstrumentation = ["dev.braintrust.instrumentation.springai.v1_0_0.auto.SpringAIAnthropicInstrumentationModule"] + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + } + pass { + group = 'org.springframework.ai' + module = 'spring-ai-anthropic' + versions = "[${springAiVersion},)" + ignoredInstrumentation = ["dev.braintrust.instrumentation.springai.v1_0_0.auto.SpringAIOpenAIInstrumentationModule"] + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + extraDependency 'com.fasterxml.jackson.datatype:jackson-datatype-jdk8' + } +} + +dependencies { + implementation project(':braintrust-java-agent:instrumenter') + implementation "io.opentelemetry:opentelemetry-api:${otelVersion}" + implementation 'com.google.code.findbugs:jsr305:3.0.2' + implementation "org.slf4j:slf4j-api:${slf4jVersion}" + implementation project(':braintrust-sdk') + + // AutoService for SPI registration + compileOnly 'com.google.auto.service:auto-service-annotations:1.1.1' + annotationProcessor 'com.google.auto.service:auto-service:1.1.1' + + // ByteBuddy for ElementMatcher types used in instrumentation definitions + compileOnly 'net.bytebuddy:byte-buddy:1.17.5' + + // Target libraries — compileOnly because they will be on the app classpath at runtime + compileOnly "org.springframework.ai:spring-ai-model:${springAiVersion}" + compileOnly "org.springframework.ai:spring-ai-openai:${springAiVersion}" + compileOnly "org.springframework.ai:spring-ai-anthropic:${springAiVersion}" + + // Test dependencies + testImplementation(testFixtures(project(":test-harness"))) + testImplementation project(':braintrust-java-agent:instrumenter') + testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" + testRuntimeOnly 'org.junit.platform:junit-platform-launcher' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.17.5' + testRuntimeOnly "org.slf4j:slf4j-simple:${slf4jVersion}" + testImplementation "org.springframework.ai:spring-ai-model:${springAiVersion}" + testImplementation "org.springframework.ai:spring-ai-openai:${springAiVersion}" + testImplementation "org.springframework.ai:spring-ai-anthropic:${springAiVersion}" + // spring-ai-openai and spring-ai-anthropic require spring-webflux at runtime for WebClient + testRuntimeOnly 'org.springframework:spring-webflux:6.2.3' + // WireMock 3.x bundles Jetty 11, but Spring WebFlux 6.2 requires Jetty 12 for its + // JettyClientHttpConnector. Adding reactor-netty-http gives WebFlux a Netty connector to + // use for streaming instead, sidestepping the Jetty version conflict entirely. + testRuntimeOnly 'io.projectreactor.netty:reactor-netty-http:1.2.3' + // Force httpclient5 version to match what spring-ai expects (WireMock pulls in an older one) + testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' + testImplementation 'org.apache.httpcomponents.core5:httpcore5:5.2.4' +} + +test { + useJUnitPlatform() + workingDir = rootProject.projectDir + testLogging { + events "passed", "skipped", "failed" + showStandardStreams = true + exceptionFormat "full" + } +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/AnthropicBuilderWrapper.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/AnthropicBuilderWrapper.java new file mode 100644 index 00000000..115ddf0b --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/AnthropicBuilderWrapper.java @@ -0,0 +1,141 @@ +package dev.braintrust.instrumentation.springai.v1_0_0; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import dev.braintrust.instrumentation.InstrumentationSemConv; +import dev.braintrust.json.BraintrustJsonMapper; +import io.micrometer.observation.ObservationRegistry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.WeakHashMap; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.anthropic.AnthropicChatModel; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.metadata.Usage; +import org.springframework.ai.chat.model.ChatResponse; + +/** Braintrust Spring AI Anthropic instrumentation entry point. */ +@Slf4j +class AnthropicBuilderWrapper { + private static final String TRACER_NAME = "braintrust-java"; + private static final Set REGISTERED_REGISTRIES = + Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())); + + /** Reflection-friendly entry point called from {@link BraintrustSpringAI#wrap}. */ + static void wrap(OpenTelemetry openTelemetry, Object builderObj) { + wrap(openTelemetry, (AnthropicChatModel.Builder) builderObj); + } + + /** Instruments an {@link AnthropicChatModel.Builder} in place before {@code build()} runs. */ + static void wrap(OpenTelemetry openTelemetry, AnthropicChatModel.Builder builder) { + try { + Tracer tracer = openTelemetry.getTracer(TRACER_NAME); + ObservationRegistry registry = getField(builder, "observationRegistry"); + if (registry == null || registry.isNoop()) { + registry = ObservationRegistry.create(); + builder.observationRegistry(registry); + } + synchronized (REGISTERED_REGISTRIES) { + if (!REGISTERED_REGISTRIES.contains(registry)) { + registry.observationConfig() + .observationHandler( + new BraintrustObservationHandler( + tracer, + AnthropicBuilderWrapper::tagSpanRequest, + AnthropicBuilderWrapper::tagSpanResponse)); + REGISTERED_REGISTRIES.add(registry); + } + } + } catch (Exception e) { + log.error("failed to prepare Spring AI Anthropic builder", e); + } + } + + // ------------------------------------------------------------------------- + // Span-tagging helpers + // ------------------------------------------------------------------------- + + @lombok.SneakyThrows + static void tagSpanRequest(Span span, org.springframework.ai.chat.prompt.Prompt prompt) { + ArrayNode messages = BraintrustJsonMapper.get().createArrayNode(); + for (Message msg : prompt.getInstructions()) { + ObjectNode msgNode = BraintrustJsonMapper.get().createObjectNode(); + msgNode.put("role", msg.getMessageType().getValue().toLowerCase()); + msgNode.put("content", msg.getText()); + messages.add(msgNode); + } + String model = null; + if (prompt.getOptions() != null && prompt.getOptions().getModel() != null) { + model = prompt.getOptions().getModel().toString(); + } + ObjectNode requestBody = BraintrustJsonMapper.get().createObjectNode(); + requestBody.set("messages", messages); + if (model != null) requestBody.put("model", model); + + InstrumentationSemConv.tagLLMSpanRequest( + span, + InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, + "https://api.anthropic.com", + List.of("v1", "messages"), + "POST", + BraintrustJsonMapper.toJson(requestBody)); + } + + @lombok.SneakyThrows + static void tagSpanResponse(Span span, ChatResponse chatResponse) { + ArrayNode content = BraintrustJsonMapper.get().createArrayNode(); + for (var generation : chatResponse.getResults()) { + ObjectNode block = BraintrustJsonMapper.get().createObjectNode(); + block.put("type", "text"); + block.put("text", generation.getOutput().getText()); + content.add(block); + } + ObjectNode responseBody = BraintrustJsonMapper.get().createObjectNode(); + responseBody.set("content", content); + + ChatResponseMetadata metadata = chatResponse.getMetadata(); + if (metadata != null && metadata.getUsage() != null) { + Usage usage = metadata.getUsage(); + Integer promptTokens = usage.getPromptTokens(); + Integer completionTokens = usage.getCompletionTokens(); + ObjectNode usageNode = BraintrustJsonMapper.get().createObjectNode(); + if (promptTokens != null) usageNode.put("input_tokens", promptTokens); + if (completionTokens != null) usageNode.put("output_tokens", completionTokens); + responseBody.set("usage", usageNode); + } + + InstrumentationSemConv.tagLLMSpanResponse( + span, + InstrumentationSemConv.PROVIDER_NAME_ANTHROPIC, + BraintrustJsonMapper.toJson(responseBody)); + } + + // ------------------------------------------------------------------------- + // Internal helpers + // ------------------------------------------------------------------------- + + @SuppressWarnings("unchecked") + private static T getField(Object obj, String fieldName) + throws ReflectiveOperationException { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(obj); + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException( + "Field '" + fieldName + "' not found on " + obj.getClass().getName()); + } + + private AnthropicBuilderWrapper() {} +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustObservationHandler.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustObservationHandler.java new file mode 100644 index 00000000..de269ee1 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustObservationHandler.java @@ -0,0 +1,76 @@ +package dev.braintrust.instrumentation.springai.v1_0_0; + +import dev.braintrust.instrumentation.InstrumentationSemConv; +import io.micrometer.observation.Observation; +import io.micrometer.observation.ObservationHandler; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import java.util.function.BiConsumer; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.observation.ChatModelObservationContext; +import org.springframework.ai.chat.prompt.Prompt; + +/** + * Provider-agnostic Micrometer observation handler for Spring AI chat model calls. + * + *

Starts an OTel span on observation start and ends it on stop/error. Provider-specific request + * and response tagging is delegated to the supplied {@code tagRequest} and {@code tagResponse} + * callbacks so that OpenAI and Anthropic can each supply the correct format. + */ +final class BraintrustObservationHandler + implements ObservationHandler { + private static final String OBSERVATION_SPAN_KEY = + BraintrustObservationHandler.class.getName() + ".span"; + + private final Tracer tracer; + private final BiConsumer tagRequest; + private final BiConsumer tagResponse; + + BraintrustObservationHandler( + Tracer tracer, + BiConsumer tagRequest, + BiConsumer tagResponse) { + this.tracer = tracer; + this.tagRequest = tagRequest; + this.tagResponse = tagResponse; + } + + @Override + public boolean supportsContext(Observation.Context context) { + return context instanceof ChatModelObservationContext; + } + + @Override + public void onStart(ChatModelObservationContext context) { + Span span = tracer.spanBuilder(InstrumentationSemConv.UNSET_LLM_SPAN_NAME).startSpan(); + context.put(OBSERVATION_SPAN_KEY, span); + Prompt prompt = context.getRequest(); + if (prompt != null) { + tagRequest.accept(span, prompt); + } + } + + @Override + public void onError(ChatModelObservationContext context) { + Span span = context.get(OBSERVATION_SPAN_KEY); + if (span != null && context.getError() != null) { + InstrumentationSemConv.tagLLMSpanResponse(span, context.getError()); + } + } + + @Override + public void onStop(ChatModelObservationContext context) { + Span span = context.get(OBSERVATION_SPAN_KEY); + if (span == null) { + return; + } + try { + ChatResponse response = context.getResponse(); + if (response != null) { + tagResponse.accept(span, response); + } + } finally { + span.end(); + } + } +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java new file mode 100644 index 00000000..bc4f0ab7 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAI.java @@ -0,0 +1,51 @@ +package dev.braintrust.instrumentation.springai.v1_0_0; + +import io.opentelemetry.api.OpenTelemetry; +import java.lang.reflect.Method; +import lombok.extern.slf4j.Slf4j; + +/** + * Braintrust Spring AI instrumentation entry point. + * + *

Accepts any Spring AI chat-model builder and instruments it in place before {@code build()} + * runs. Provider-specific logic lives in {@link OpenAIBuilderWrapper} and {@link + * AnthropicBuilderWrapper}, which are only referenced here by string class name so that muzzle does + * not follow the reference when a given provider library is absent from the classpath. + */ +@Slf4j +public class BraintrustSpringAI { + private static final String OPENAI_BUILDER_CLASS = + "org.springframework.ai.openai.OpenAiChatModel$Builder"; + private static final String ANTHROPIC_BUILDER_CLASS = + "org.springframework.ai.anthropic.AnthropicChatModel$Builder"; + + private static final String OPENAI_WRAPPER_CLASS = + "dev.braintrust.instrumentation.springai.v1_0_0.OpenAIBuilderWrapper"; + private static final String ANTHROPIC_WRAPPER_CLASS = + "dev.braintrust.instrumentation.springai.v1_0_0.AnthropicBuilderWrapper"; + + /** Instruments a Spring AI chat-model builder in place. */ + public static T wrap(OpenTelemetry openTelemetry, T chatModelBuilder) { + try { + String builderClassName = chatModelBuilder.getClass().getName(); + String wrapperClass; + if (OPENAI_BUILDER_CLASS.equals(builderClassName)) { + wrapperClass = OPENAI_WRAPPER_CLASS; + } else if (ANTHROPIC_BUILDER_CLASS.equals(builderClassName)) { + wrapperClass = ANTHROPIC_WRAPPER_CLASS; + } else { + log.info("BraintrustSpringAI.wrap: unrecognised builder type {}", builderClassName); + return chatModelBuilder; + } + Class wrapper = chatModelBuilder.getClass().getClassLoader().loadClass(wrapperClass); + Method wrapMethod = + wrapper.getDeclaredMethod("wrap", OpenTelemetry.class, Object.class); + wrapMethod.invoke(null, openTelemetry, chatModelBuilder); + } catch (Exception e) { + log.error("failed to apply spring ai instrumentation", e); + } + return chatModelBuilder; + } + + private BraintrustSpringAI() {} +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/OpenAIBuilderWrapper.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/OpenAIBuilderWrapper.java new file mode 100644 index 00000000..2744e427 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/OpenAIBuilderWrapper.java @@ -0,0 +1,163 @@ +package dev.braintrust.instrumentation.springai.v1_0_0; + +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import dev.braintrust.instrumentation.InstrumentationSemConv; +import dev.braintrust.json.BraintrustJsonMapper; +import io.micrometer.observation.ObservationRegistry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.WeakHashMap; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.metadata.ChatResponseMetadata; +import org.springframework.ai.chat.metadata.Usage; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.openai.OpenAiChatModel; + +/** Braintrust Spring AI instrumentation entry point. */ +@Slf4j +class OpenAIBuilderWrapper { + private static final String TRACER_NAME = "braintrust-java"; + private static final Set REGISTERED_REGISTRIES = + Collections.synchronizedSet(Collections.newSetFromMap(new WeakHashMap<>())); + + /** Reflection-friendly entry point called from {@link BraintrustSpringAI#wrap}. */ + static void wrap(OpenTelemetry openTelemetry, Object builderObj) { + wrap(openTelemetry, (OpenAiChatModel.Builder) builderObj); + } + + /** Instruments an {@link OpenAiChatModel.Builder} in place before {@code build()} runs. */ + static OpenAiChatModel.Builder wrap( + OpenTelemetry openTelemetry, OpenAiChatModel.Builder builder) { + try { + Tracer tracer = openTelemetry.getTracer(TRACER_NAME); + ObservationRegistry registry = getField(builder, "observationRegistry"); + if (registry == null || registry.isNoop()) { + registry = ObservationRegistry.create(); + builder.observationRegistry(registry); + } + synchronized (REGISTERED_REGISTRIES) { + if (!REGISTERED_REGISTRIES.contains(registry)) { + registry.observationConfig() + .observationHandler( + new BraintrustObservationHandler( + tracer, + OpenAIBuilderWrapper::tagSpanRequest, + OpenAIBuilderWrapper::tagSpanResponse)); + REGISTERED_REGISTRIES.add(registry); + } + } + } catch (Exception e) { + log.error("failed to prepare Spring AI builder", e); + } + return builder; + } + + // ------------------------------------------------------------------------- + // Shared span-tagging helpers (used by both the observation handler and wrapStream) + // ------------------------------------------------------------------------- + + @SneakyThrows + static void tagSpanRequest(Span span, Prompt prompt) { + ArrayNode messages = BraintrustJsonMapper.get().createArrayNode(); + for (Message msg : prompt.getInstructions()) { + ObjectNode msgNode = BraintrustJsonMapper.get().createObjectNode(); + msgNode.put("role", msg.getMessageType().getValue().toLowerCase()); + msgNode.put("content", msg.getText()); + messages.add(msgNode); + } + + String model = null; + if (prompt.getOptions() != null) { + Object modelOpt = prompt.getOptions().getModel(); + if (modelOpt != null) { + model = modelOpt.toString(); + } + } + + ObjectNode requestBody = BraintrustJsonMapper.get().createObjectNode(); + requestBody.set("messages", messages); + if (model != null) { + requestBody.put("model", model); + } + + InstrumentationSemConv.tagLLMSpanRequest( + span, + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + "https://api.openai.com", + List.of("v1", "chat", "completions"), + "POST", + BraintrustJsonMapper.toJson(requestBody)); + } + + @SneakyThrows + static void tagSpanResponse(Span span, ChatResponse chatResponse) { + ArrayNode choices = BraintrustJsonMapper.get().createArrayNode(); + for (var generation : chatResponse.getResults()) { + ObjectNode choice = BraintrustJsonMapper.get().createObjectNode(); + ObjectNode message = BraintrustJsonMapper.get().createObjectNode(); + message.put("role", "assistant"); + message.put("content", generation.getOutput().getText()); + choice.set("message", message); + choice.put( + "finish_reason", + generation.getMetadata().getFinishReason() != null + ? generation.getMetadata().getFinishReason().toLowerCase() + : "stop"); + choices.add(choice); + } + + ObjectNode responseBody = BraintrustJsonMapper.get().createObjectNode(); + responseBody.set("choices", choices); + + ChatResponseMetadata metadata = chatResponse.getMetadata(); + if (metadata != null && metadata.getUsage() != null) { + Usage usage = metadata.getUsage(); + Integer promptTokens = usage.getPromptTokens(); + Integer completionTokens = usage.getCompletionTokens(); + ObjectNode usageNode = BraintrustJsonMapper.get().createObjectNode(); + if (promptTokens != null) usageNode.put("prompt_tokens", promptTokens); + if (completionTokens != null) usageNode.put("completion_tokens", completionTokens); + if (promptTokens != null && completionTokens != null) { + usageNode.put("total_tokens", promptTokens + completionTokens); + } + responseBody.set("usage", usageNode); + } + + InstrumentationSemConv.tagLLMSpanResponse( + span, + InstrumentationSemConv.PROVIDER_NAME_OPENAI, + BraintrustJsonMapper.toJson(responseBody)); + } + + // ------------------------------------------------------------------------- + // Internal helpers + // ------------------------------------------------------------------------- + + @SuppressWarnings("unchecked") + private static T getField(Object obj, String fieldName) + throws ReflectiveOperationException { + Class clazz = obj.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(obj); + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new NoSuchFieldException( + "Field '" + fieldName + "' not found on " + obj.getClass().getName()); + } + + private OpenAIBuilderWrapper() {} +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java new file mode 100644 index 00000000..5cb0d0f7 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIAnthropicInstrumentationModule.java @@ -0,0 +1,61 @@ +package dev.braintrust.instrumentation.springai.v1_0_0.auto; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import dev.braintrust.instrumentation.InstrumentationModule; +import dev.braintrust.instrumentation.TypeInstrumentation; +import dev.braintrust.instrumentation.TypeTransformer; +import dev.braintrust.instrumentation.springai.v1_0_0.BraintrustSpringAI; +import io.opentelemetry.api.GlobalOpenTelemetry; +import java.util.List; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +@AutoService(InstrumentationModule.class) +public class SpringAIAnthropicInstrumentationModule extends InstrumentationModule { + private static final String PACKAGE = "dev.braintrust.instrumentation.springai.v1_0_0."; + + public SpringAIAnthropicInstrumentationModule() { + super("springai_anthropic_1_0_0"); + } + + @Override + public List getHelperClassNames() { + return List.of( + PACKAGE + "BraintrustSpringAI", + PACKAGE + "AnthropicBuilderWrapper", + PACKAGE + "BraintrustObservationHandler", + "dev.braintrust.json.BraintrustJsonMapper", + "dev.braintrust.instrumentation.InstrumentationSemConv"); + } + + @Override + public List typeInstrumentations() { + return List.of(new AnthropicChatModelBuilderInstrumentation()); + } + + public static class AnthropicChatModelBuilderInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("org.springframework.ai.anthropic.AnthropicChatModel$Builder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + SpringAIAnthropicInstrumentationModule.class.getName() + + "$AnthropicChatModelBuilderAdvice"); + } + } + + private static class AnthropicChatModelBuilderAdvice { + @Advice.OnMethodEnter + public static void build(@Advice.This Object builder) { + BraintrustSpringAI.wrap(GlobalOpenTelemetry.get(), builder); + } + } +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java new file mode 100644 index 00000000..3a3f76a3 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/main/java/dev/braintrust/instrumentation/springai/v1_0_0/auto/SpringAIOpenAIInstrumentationModule.java @@ -0,0 +1,62 @@ +package dev.braintrust.instrumentation.springai.v1_0_0.auto; + +import static net.bytebuddy.matcher.ElementMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import dev.braintrust.instrumentation.InstrumentationModule; +import dev.braintrust.instrumentation.TypeInstrumentation; +import dev.braintrust.instrumentation.TypeTransformer; +import dev.braintrust.instrumentation.springai.v1_0_0.BraintrustSpringAI; +import io.opentelemetry.api.GlobalOpenTelemetry; +import java.util.List; +import net.bytebuddy.asm.Advice; +import net.bytebuddy.description.type.TypeDescription; +import net.bytebuddy.matcher.ElementMatcher; + +@AutoService(InstrumentationModule.class) +public class SpringAIOpenAIInstrumentationModule extends InstrumentationModule { + private static final String PACKAGE = "dev.braintrust.instrumentation.springai.v1_0_0."; + + public SpringAIOpenAIInstrumentationModule() { + super("springai_openai_1_0_0"); + } + + @Override + public List getHelperClassNames() { + return List.of( + PACKAGE + "BraintrustSpringAI", + PACKAGE + "OpenAIBuilderWrapper", + PACKAGE + "BraintrustObservationHandler", + "dev.braintrust.json.BraintrustJsonMapper", + "dev.braintrust.instrumentation.InstrumentationSemConv"); + } + + @Override + public List typeInstrumentations() { + return List.of(new OpenAiChatModelBuilderInstrumentation()); + } + + /** Intercepts {@code OpenAiChatModel.Builder.build()} to inject instrumentation. */ + public static class OpenAiChatModelBuilderInstrumentation implements TypeInstrumentation { + @Override + public ElementMatcher typeMatcher() { + return named("org.springframework.ai.openai.OpenAiChatModel$Builder"); + } + + @Override + public void transform(TypeTransformer transformer) { + transformer.applyAdviceToMethod( + named("build").and(takesArguments(0)), + SpringAIOpenAIInstrumentationModule.class.getName() + + "$OpenAiChatModelBuilderAdvice"); + } + } + + private static class OpenAiChatModelBuilderAdvice { + @Advice.OnMethodEnter + public static void build(@Advice.This Object builder) { + BraintrustSpringAI.wrap(GlobalOpenTelemetry.get(), builder); + } + } +} diff --git a/braintrust-java-agent/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java b/braintrust-java-agent/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java new file mode 100644 index 00000000..948b0423 --- /dev/null +++ b/braintrust-java-agent/instrumentation/springai_1_0_0/src/test/java/dev/braintrust/instrumentation/springai/v1_0_0/BraintrustSpringAITest.java @@ -0,0 +1,314 @@ +package dev.braintrust.instrumentation.springai.v1_0_0; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.braintrust.TestHarness; +import dev.braintrust.instrumentation.Instrumenter; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.trace.data.SpanData; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import net.bytebuddy.agent.ByteBuddyAgent; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.ai.anthropic.AnthropicChatModel; +import org.springframework.ai.anthropic.AnthropicChatOptions; +import org.springframework.ai.anthropic.api.AnthropicApi; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.openai.OpenAiChatModel; +import org.springframework.ai.openai.OpenAiChatOptions; +import org.springframework.ai.openai.api.OpenAiApi; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class BraintrustSpringAITest { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + @BeforeAll + public void beforeAll() { + var instrumentation = ByteBuddyAgent.install(); + Instrumenter.install(instrumentation, BraintrustSpringAITest.class.getClassLoader()); + } + + private TestHarness testHarness; + + @BeforeEach + void beforeEach() { + testHarness = TestHarness.setup(); + } + + // ------------------------------------------------------------------------- + // Provider descriptor — carries only name and expected assertions. + // ChatModel is built fresh per test via buildChatModel() so it uses the + // current testHarness's OpenTelemetry instance. + // ------------------------------------------------------------------------- + + record Provider( + String name, + String expectedProvider, + String expectedModelPrefix, + boolean outputIsChoicesArray) { + @Override + public String toString() { + return name; + } + } + + static Stream providers() { + return Stream.of( + new Provider("openai", "openai", "gpt-4o-mini", true), + new Provider("anthropic", "anthropic", "claude-3-haiku", false)); + } + + /** Builds a fresh {@link ChatModel} for each test so it picks up the current OTel instance. */ + private ChatModel buildChatModel(Provider provider) { + return switch (provider.name()) { + case "openai" -> { + // testHarness.openAiBaseUrl() returns a URL ending in "/v1" (both the real API + // and the VCR proxy). Spring AI's default completionsPath is "/v1/chat/completions" + // which would double the "/v1". Override it to just "/chat/completions" so the + // full URL resolves correctly in all VCR modes. + var api = + OpenAiApi.builder() + .baseUrl(testHarness.openAiBaseUrl()) + .completionsPath("/chat/completions") + .apiKey(testHarness.openAiApiKey()) + .build(); + yield OpenAiChatModel.builder() + .openAiApi(api) + .defaultOptions( + OpenAiChatOptions.builder() + .model("gpt-4o-mini") + .temperature(0.0) + .maxTokens(50) + .build()) + .build(); + } + case "anthropic" -> { + var api = + AnthropicApi.builder() + .baseUrl(testHarness.anthropicBaseUrl()) + .apiKey(testHarness.anthropicApiKey()) + .build(); + yield AnthropicChatModel.builder() + .anthropicApi(api) + .defaultOptions( + AnthropicChatOptions.builder() + .model("claude-3-haiku-20240307") + .temperature(0.0) + .maxTokens(50) + .build()) + .build(); + } + default -> throw new IllegalArgumentException("Unknown provider: " + provider.name()); + }; + } + + // ------------------------------------------------------------------------- + // Tests + // ------------------------------------------------------------------------- + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + @SneakyThrows + void testCall(Provider provider) { + ChatModel chatModel = buildChatModel(provider); + var response = chatModel.call(new Prompt("What is the capital of France?")); + + assertNotNull(response); + String text = response.getResult().getOutput().getText(); + assertTrue(text.toLowerCase().contains("paris"), "Response should mention Paris: " + text); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + SpanData span = spans.get(0); + + assertCommonSpanAttributes(span, provider); + assertInputMessages(span, 1); + assertEquals("user", inputMessages(span).get(0).get("role").asText()); + assertOutputMentionsParis(span, provider); + assertTokenMetrics(span); + assertFalse( + metrics(span).has("time_to_first_token"), + "time_to_first_token should not be present for non-streaming"); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + @SneakyThrows + void testCallWithSystemMessage(Provider provider) { + ChatModel chatModel = buildChatModel(provider); + var prompt = + new Prompt( + java.util.List.of( + new SystemMessage("You are a helpful geography assistant."), + new UserMessage("What is the capital of France?"))); + var response = chatModel.call(prompt); + + assertNotNull(response); + String text = response.getResult().getOutput().getText(); + assertTrue(text.toLowerCase().contains("paris"), "Response should mention Paris: " + text); + + var spans = testHarness.awaitExportedSpans(); + assertEquals(1, spans.size()); + SpanData span = spans.get(0); + + assertCommonSpanAttributes(span, provider); + assertInputMessages(span, 2); + JsonNode messages = inputMessages(span); + assertEquals("system", messages.get(0).get("role").asText()); + assertEquals("user", messages.get(1).get("role").asText()); + assertTrue( + messages.get(1).get("content").asText().contains("capital"), + "user message should contain the prompt text"); + assertOutputMentionsParis(span, provider); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("providers") + @SneakyThrows + void testStream(Provider provider) { + ChatModel chatModel = buildChatModel(provider); + var fullText = new StringBuilder(); + chatModel.stream(streamPrompt(provider)) + .doOnNext( + chunk -> { + if (chunk.getResult() != null + && chunk.getResult().getOutput() != null + && chunk.getResult().getOutput().getText() != null) { + fullText.append(chunk.getResult().getOutput().getText()); + } + }) + .blockLast(); + + assertFalse(fullText.isEmpty(), "Should have received streaming chunks"); + assertTrue( + fullText.toString().toLowerCase().contains("paris"), + "Streamed response should mention Paris: " + fullText); + + // Observation span completes on Reactor scheduler thread; wait for it + var spans = testHarness.awaitExportedSpans(1); + assertEquals(1, spans.size()); + SpanData span = spans.get(0); + + assertCommonSpanAttributes(span, provider); + assertInputMessages(span, 1); + assertEquals("user", inputMessages(span).get(0).get("role").asText()); + assertOutputMentionsParis(span, provider); + assertTokenMetrics(span); + } + + // ------------------------------------------------------------------------- + // Shared assertion helpers + // ------------------------------------------------------------------------- + + @SneakyThrows + private void assertCommonSpanAttributes(SpanData span, Provider provider) { + assertEquals("llm", spanAttributes(span).get("type").asText()); + assertEquals(provider.expectedProvider(), metadata(span).get("provider").asText()); + assertTrue( + metadata(span).get("model").asText().startsWith(provider.expectedModelPrefix()), + "model should start with " + + provider.expectedModelPrefix() + + ", got: " + + metadata(span).get("model").asText()); + } + + @SneakyThrows + private void assertInputMessages(SpanData span, int expectedCount) { + assertTrue(inputMessages(span).isArray(), "input_json should be an array"); + assertEquals( + expectedCount, + inputMessages(span).size(), + "Expected " + expectedCount + " input message(s)"); + } + + @SneakyThrows + private void assertOutputMentionsParis(SpanData span, Provider provider) { + String outputJson = + span.getAttributes().get(AttributeKey.stringKey("braintrust.output_json")); + assertNotNull(outputJson, "braintrust.output_json should be set"); + JsonNode output = JSON_MAPPER.readTree(outputJson); + + String assistantText; + if (provider.outputIsChoicesArray()) { + assertTrue(output.isArray(), "output_json should be an array for " + provider.name()); + assertTrue(output.size() > 0); + assistantText = output.get(0).get("message").get("content").asText(); + } else { + assertTrue( + output.has("content"), + "output_json should have content field for " + provider.name()); + assistantText = output.get("content").get(0).get("text").asText(); + } + assertTrue( + assistantText.toLowerCase().contains("paris"), + "Output should mention Paris for " + provider.name() + ": " + assistantText); + } + + private Prompt streamPrompt(Provider provider) { + if ("openai".equals(provider.name())) { + return new Prompt( + "What is the capital of France?", + OpenAiChatOptions.builder() + .model("gpt-4o-mini") + .temperature(0.0) + .maxTokens(50) + .streamUsage(true) + .build()); + } + return new Prompt("What is the capital of France?"); + } + + private void assertTokenMetrics(SpanData span) { + JsonNode m = metrics(span); + assertTrue(m.has("prompt_tokens"), "prompt_tokens should be present"); + assertTrue(m.get("prompt_tokens").asInt() > 0, "prompt_tokens should be positive"); + assertTrue(m.has("completion_tokens"), "completion_tokens should be present"); + assertTrue(m.get("completion_tokens").asInt() > 0, "completion_tokens should be positive"); + if (m.has("prompt_tokens") && m.has("completion_tokens")) { + assertTrue(m.has("tokens"), "tokens should be present when prompt+completion are"); + assertTrue(m.get("tokens").asInt() > 0, "tokens should be positive"); + } + } + + // ------------------------------------------------------------------------- + // Attribute extractors + // ------------------------------------------------------------------------- + + @SneakyThrows + private JsonNode spanAttributes(SpanData span) { + String json = + span.getAttributes().get(AttributeKey.stringKey("braintrust.span_attributes")); + assertNotNull(json, "braintrust.span_attributes should be set"); + return JSON_MAPPER.readTree(json); + } + + @SneakyThrows + private JsonNode metadata(SpanData span) { + String json = span.getAttributes().get(AttributeKey.stringKey("braintrust.metadata")); + assertNotNull(json, "braintrust.metadata should be set"); + return JSON_MAPPER.readTree(json); + } + + @SneakyThrows + private JsonNode inputMessages(SpanData span) { + String json = span.getAttributes().get(AttributeKey.stringKey("braintrust.input_json")); + assertNotNull(json, "braintrust.input_json should be set"); + return JSON_MAPPER.readTree(json); + } + + @SneakyThrows + private JsonNode metrics(SpanData span) { + String json = span.getAttributes().get(AttributeKey.stringKey("braintrust.metrics")); + assertNotNull(json, "braintrust.metrics should be set"); + return JSON_MAPPER.readTree(json); + } +} diff --git a/braintrust-sdk/build.gradle b/braintrust-sdk/build.gradle index df795b5b..4ce6f825 100644 --- a/braintrust-sdk/build.gradle +++ b/braintrust-sdk/build.gradle @@ -51,7 +51,7 @@ dependencies { testImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}" testImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}" testImplementation "org.junit.jupiter:junit-jupiter-params:${junitVersion}" - testImplementation 'com.github.tomakehurst:wiremock-jre8:2.35.0' + testImplementation 'org.wiremock:wiremock:3.13.1' // OAI instrumentation compileOnly 'com.openai:openai-java:2.8.1' diff --git a/examples/build.gradle b/examples/build.gradle index 7ed05420..fe166f90 100644 --- a/examples/build.gradle +++ b/examples/build.gradle @@ -30,7 +30,11 @@ dependencies { // to run gemini examples implementation 'com.google.genai:google-genai:1.20.0' // spring ai examples + implementation 'org.springframework.ai:spring-ai-anthropic:1.1.0' implementation 'org.springframework.ai:spring-ai-google-genai:1.1.0' + implementation 'org.springframework.ai:spring-ai-openai:1.1.0' + // spring-ai-openai requires spring-webflux (WebClient) at runtime + implementation 'org.springframework:spring-webflux:6.2.3' // spring boot for SpringAIExample (exclude logback, use slf4j-simple like other examples) implementation('org.springframework.boot:spring-boot-starter:3.4.1') { exclude group: 'org.springframework.boot', module: 'spring-boot-starter-logging' @@ -133,10 +137,11 @@ task runGeminiInstrumentation(type: JavaExec) { task runSpringAI(type: JavaExec) { group = 'Braintrust SDK Examples' - description = 'Run the Spring Boot + Spring AI + Gemini example.' + description = 'Run the Spring Boot + Spring AI example' classpath = sourceSets.main.runtimeClasspath mainClass = 'dev.braintrust.examples.SpringAIExample' systemProperty 'org.slf4j.simpleLogger.log.dev.braintrust', braintrustLogLevel + debugOptions { enabled = true port = 5566 diff --git a/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java b/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java index dfdfc3a3..ad14c1c1 100644 --- a/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java +++ b/examples/src/main/java/dev/braintrust/examples/SpringAIExample.java @@ -8,10 +8,16 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; import io.opentelemetry.context.Scope; +import org.springframework.ai.anthropic.AnthropicChatModel; +import org.springframework.ai.anthropic.AnthropicChatOptions; +import org.springframework.ai.anthropic.api.AnthropicApi; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.google.genai.GoogleGenAiChatModel; import org.springframework.ai.google.genai.GoogleGenAiChatOptions; +import org.springframework.ai.openai.OpenAiChatModel; +import org.springframework.ai.openai.OpenAiChatOptions; +import org.springframework.ai.openai.api.OpenAiApi; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -27,7 +33,9 @@ public class SpringAIExample { public static void main(String[] args) { - SpringApplication.run(SpringAIExample.class, args); + var app = new SpringApplication(SpringAIExample.class); + app.setWebApplicationType(org.springframework.boot.WebApplicationType.NONE); + app.run(args); } @Bean @@ -77,16 +85,60 @@ public Tracer tracer(OpenTelemetry openTelemetry) { @Bean public String aiProvider() { - // return "openai"; - // return "anthropic"; - return "google"; + var provider = System.getenv("SPRING_AI_EXAMPLE_PROVIDER"); + if (provider == null || provider.isBlank()) { + return "openai"; + } + return switch (provider) { + case "openai", "anthropic", "google" -> provider; + default -> + throw new RuntimeException( + "unsupported SPRING_AI_EXAMPLE_PROVIDER: '%s'. Allowed values: openai, anthropic, google" + .formatted(provider)); + }; } @Bean public ChatModel chatModel(String aiProvider, OpenTelemetry openTelemetry) { return switch (aiProvider) { - case "openai", "anthropic" -> { - throw new RuntimeException("TODO: " + aiProvider); + case "openai" -> { + if (null == System.getenv("OPENAI_API_KEY")) { + System.err.println( + "\n" + + "WARNING: OPENAI_API_KEY not found. This example will likely" + + " fail.\n" + + "Set it with: export OPENAI_API_KEY='your-key'\n"); + } + var openAiApi = OpenAiApi.builder().apiKey(System.getenv("OPENAI_API_KEY")).build(); + yield OpenAiChatModel.builder() + .openAiApi(openAiApi) + .defaultOptions( + OpenAiChatOptions.builder() + .model("gpt-4o-mini") + .temperature(0.0) + .maxTokens(50) + .build()) + .build(); + } + case "anthropic" -> { + if (null == System.getenv("ANTHROPIC_API_KEY")) { + System.err.println( + "\n" + + "WARNING: ANTHROPIC_API_KEY not found. This example will" + + " likely fail.\n" + + "Set it with: export ANTHROPIC_API_KEY='your-key'\n"); + } + var anthropicApi = + AnthropicApi.builder().apiKey(System.getenv("ANTHROPIC_API_KEY")).build(); + yield AnthropicChatModel.builder() + .anthropicApi(anthropicApi) + .defaultOptions( + AnthropicChatOptions.builder() + .model("claude-3-haiku-20240307") + .temperature(0.0) + .maxTokens(50) + .build()) + .build(); } case "google" -> { if (null == System.getenv("GOOGLE_API_KEY") diff --git a/settings.gradle b/settings.gradle index a57eb0aa..fae6cad9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,6 +15,7 @@ include 'braintrust-java-agent:instrumentation:openai_2_8_0' include 'braintrust-java-agent:instrumentation:anthropic_2_2_0' include 'braintrust-java-agent:instrumentation:genai_1_18_0' include 'braintrust-java-agent:instrumentation:langchain_1_8_0' +include 'braintrust-java-agent:instrumentation:springai_1_0_0' include 'braintrust-java-agent:smoke-test:test-instrumentation' include 'braintrust-java-agent:smoke-test:dd-agent' include 'braintrust-java-agent:smoke-test:otel-agent' diff --git a/test-harness/build.gradle b/test-harness/build.gradle index 8346e16f..67068d96 100644 --- a/test-harness/build.gradle +++ b/test-harness/build.gradle @@ -34,7 +34,7 @@ dependencies { testFixturesImplementation "io.opentelemetry:opentelemetry-sdk-logs:${otelVersion}" testFixturesImplementation "io.opentelemetry:opentelemetry-sdk-testing:${otelVersion}" - testFixturesImplementation 'com.github.tomakehurst:wiremock-jre8:2.35.0' + testFixturesImplementation 'org.wiremock:wiremock:3.13.1' testFixturesImplementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" testFixturesImplementation "org.junit.jupiter:junit-jupiter:${junitVersion}"