diff --git a/e2e/helpers/anthropic-trace-contract.ts b/e2e/helpers/anthropic-trace-contract.ts deleted file mode 100644 index ef7f27294..000000000 --- a/e2e/helpers/anthropic-trace-contract.ts +++ /dev/null @@ -1,269 +0,0 @@ -import { expect } from "vitest"; -import { normalizeForSnapshot, type Json } from "./normalize"; -import type { - CapturedLogEvent, - CapturedLogPayload, -} from "./mock-braintrust-server"; -import { findChildSpans, findLatestSpan } from "./trace-selectors"; -import { - payloadRowsForRootSpan, - summarizeWrapperContract, -} from "./wrapper-contract"; - -function findNamedChildSpan( - capturedEvents: CapturedLogEvent[], - names: string[], - parentId: string | undefined, -) { - for (const name of names) { - const span = findChildSpans(capturedEvents, name, parentId)[0]; - if (span) { - return span; - } - } - - return undefined; -} - -function normalizeAnthropicPayloads(payloadRows: unknown[]): unknown[] { - const attachmentRowKeys = new Set(); - - for (const payload of payloadRows) { - if (!payload || typeof payload !== "object") { - continue; - } - - const row = payload as { - id?: string; - input?: Array<{ - content?: - | string - | Array<{ - source?: { - data?: { - type?: string; - }; - }; - }>; - }>; - span_id?: string; - }; - - const hasAttachmentInput = row.input?.some( - (message) => - Array.isArray(message.content) && - message.content.some( - (block) => block.source?.data?.type === "braintrust_attachment", - ), - ); - - if (!hasAttachmentInput) { - continue; - } - - if (typeof row.id === "string") { - attachmentRowKeys.add(row.id); - } - if (typeof row.span_id === "string") { - attachmentRowKeys.add(row.span_id); - } - } - - return payloadRows.map((payload) => { - if (!payload || typeof payload !== "object") { - return payload; - } - - const row = structuredClone(payload) as { - id?: string; - metadata?: { operation?: string }; - output?: { - content?: Array<{ text?: string; type?: string }>; - }; - span_id?: string; - }; - const isAttachmentRow = - row.metadata?.operation === "attachment" || - (typeof row.id === "string" && attachmentRowKeys.has(row.id)) || - (typeof row.span_id === "string" && attachmentRowKeys.has(row.span_id)); - - if (isAttachmentRow) { - const textBlock = row.output?.content?.find( - (block) => block.type === "text" && typeof block.text === "string", - ); - if (textBlock) { - textBlock.text = ""; - } - } - - return row; - }); -} - -export function assertAnthropicTraceContract(options: { - capturedEvents: CapturedLogEvent[]; - payloads: CapturedLogPayload[]; - rootName: string; - scenarioName: string; -}): { payloadSummary: Json; spanSummary: Json } { - const root = findLatestSpan(options.capturedEvents, options.rootName); - const createOperation = findLatestSpan( - options.capturedEvents, - "anthropic-create-operation", - ); - const streamOperation = findLatestSpan( - options.capturedEvents, - "anthropic-stream-operation", - ); - const withResponseOperation = findLatestSpan( - options.capturedEvents, - "anthropic-stream-with-response-operation", - ); - const toolOperation = findLatestSpan( - options.capturedEvents, - "anthropic-tool-operation", - ); - const attachmentOperation = findLatestSpan( - options.capturedEvents, - "anthropic-attachment-operation", - ); - const betaCreateOperation = findLatestSpan( - options.capturedEvents, - "anthropic-beta-create-operation", - ); - const betaStreamOperation = findLatestSpan( - options.capturedEvents, - "anthropic-beta-stream-operation", - ); - - expect(root).toBeDefined(); - expect(createOperation).toBeDefined(); - expect(streamOperation).toBeDefined(); - expect(withResponseOperation).toBeDefined(); - expect(toolOperation).toBeDefined(); - expect(attachmentOperation).toBeDefined(); - expect(betaCreateOperation).toBeDefined(); - expect(betaStreamOperation).toBeDefined(); - - expect(root?.row.metadata).toMatchObject({ - scenario: options.scenarioName, - }); - expect(createOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(streamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(withResponseOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(toolOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(attachmentOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(betaCreateOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - expect(betaStreamOperation?.span.parentIds).toEqual([root?.span.id ?? ""]); - - const createSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create"], - createOperation?.span.id, - ); - const streamSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create"], - streamOperation?.span.id, - ); - const withResponseSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create"], - withResponseOperation?.span.id, - ); - const toolSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create"], - toolOperation?.span.id, - ); - const attachmentSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create"], - attachmentOperation?.span.id, - ); - const betaCreateSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create", "anthropic.beta.messages.create"], - betaCreateOperation?.span.id, - ); - const betaStreamSpan = findNamedChildSpan( - options.capturedEvents, - ["anthropic.messages.create", "anthropic.beta.messages.create"], - betaStreamOperation?.span.id, - ); - - for (const wrapperSpan of [ - createSpan, - streamSpan, - withResponseSpan, - toolSpan, - attachmentSpan, - betaCreateSpan, - betaStreamSpan, - ]) { - expect(wrapperSpan).toBeDefined(); - expect(wrapperSpan?.row.metadata).toMatchObject({ - provider: "anthropic", - }); - expect( - typeof (wrapperSpan?.row.metadata as { model?: unknown } | undefined) - ?.model, - ).toBe("string"); - } - - for (const streamingSpan of [streamSpan, withResponseSpan, betaStreamSpan]) { - expect(streamingSpan?.metrics).toMatchObject({ - time_to_first_token: expect.any(Number), - prompt_tokens: expect.any(Number), - completion_tokens: expect.any(Number), - }); - } - - const toolOutput = toolSpan?.output as - | { - content?: Array<{ name?: string; type?: string }>; - } - | undefined; - expect( - toolOutput?.content?.some( - (block) => block.type === "tool_use" && block.name === "get_weather", - ), - ).toBe(true); - - const attachmentInput = JSON.stringify(attachmentSpan?.input); - expect(attachmentInput).toContain("image.png"); - - return { - spanSummary: normalizeForSnapshot( - [ - root, - createOperation, - createSpan, - attachmentOperation, - attachmentSpan, - streamOperation, - streamSpan, - withResponseOperation, - withResponseSpan, - toolOperation, - toolSpan, - betaCreateOperation, - betaCreateSpan, - betaStreamOperation, - betaStreamSpan, - ].map((event) => - summarizeWrapperContract(event!, [ - "provider", - "model", - "operation", - "scenario", - ]), - ) as Json, - ), - payloadSummary: normalizeForSnapshot( - normalizeAnthropicPayloads( - payloadRowsForRootSpan(options.payloads, root?.span.id), - ) as Json, - ), - }; -} diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json deleted file mode 100644 index 71bb1bc4b..000000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/log-payloads.json +++ /dev/null @@ -1,978 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "anthropic-auto-instrumentation-node-hook", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "anthropic-auto-hook-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "anthropic-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "time_to_first_token": 0, - "tokens": 17 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "OK.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "attachment", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "anthropic-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 29, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "time_to_first_token": 0, - "tokens": 1418 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "anthropic-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-with-response", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "anthropic-stream-with-response-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "anthropic-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 128, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0, - "tools": [ - { - "description": "Get the current weather in a given location", - "input_schema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - }, - "name": "get_weather" - } - ] - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 55, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 357, - "time_to_first_token": 0, - "tokens": 412 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "tool_use", - "stop_sequence": null - }, - "output": { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "anthropic-beta-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "anthropic.beta.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "time_to_first_token": 0, - "tokens": 19 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "BETA.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 13, - "name": "anthropic-beta-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "context": {}, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 14, - "name": "anthropic.beta.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/package.json b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/package.json deleted file mode 100644 index 71f92e086..000000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@braintrust/e2e-anthropic-auto-instrumentation-node-hook", - "private": true, - "braintrustScenario": { - "canary": { - "dependencies": { - "@anthropic-ai/sdk": "latest" - } - } - }, - "dependencies": { - "@anthropic-ai/sdk": "0.60.0" - } -} diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/pnpm-lock.yaml b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/pnpm-lock.yaml deleted file mode 100644 index ea50cf7ab..000000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/pnpm-lock.yaml +++ /dev/null @@ -1,23 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - '@anthropic-ai/sdk': - specifier: 0.60.0 - version: 0.60.0 - -packages: - - '@anthropic-ai/sdk@0.60.0': - resolution: {integrity: sha512-9zu/TXaUy8BZhXedDtt1wT3H4LOlpKDO1/ftiFpeR3N1PCr3KJFKkxxlQWWt1NNp08xSwUNJ3JNY8yhl8av6eQ==} - hasBin: true - -snapshots: - - '@anthropic-ai/sdk@0.60.0': {} diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs deleted file mode 100644 index 43e01c4fd..000000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.mjs +++ /dev/null @@ -1,14 +0,0 @@ -import Anthropic from "@anthropic-ai/sdk"; -import { runAnthropicScenario } from "../../helpers/anthropic-scenario.mjs"; -import { runMain } from "../../helpers/provider-runtime.mjs"; - -runMain(async () => - runAnthropicScenario({ - Anthropic, - projectNameBase: "e2e-anthropic-auto-instrumentation-hook", - rootName: "anthropic-auto-hook-root", - scenarioName: "anthropic-auto-instrumentation-node-hook", - testImageUrl: new URL("./test-image.png", import.meta.url), - useMessagesStreamHelper: false, - }), -); diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts b/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts deleted file mode 100644 index 15a80302c..000000000 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/scenario.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { expect, test } from "vitest"; -import { assertAnthropicTraceContract } from "../../helpers/anthropic-trace-contract"; -import { - formatJsonFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; -import { - prepareScenarioDir, - resolveScenarioDir, - withScenarioHarness, -} from "../../helpers/scenario-harness"; -import { E2E_TAGS } from "../../helpers/tags"; - -const scenarioDir = await prepareScenarioDir({ - scenarioDir: resolveScenarioDir(import.meta.url), -}); -const TIMEOUT_MS = 90_000; - -test( - "anthropic auto-instrumentation via node hook collects the shared anthropic trace contract", - { - tags: [E2E_TAGS.externalApi], - timeout: TIMEOUT_MS, - }, - async () => { - await withScenarioHarness( - async ({ events, payloads, runNodeScenarioDir }) => { - await runNodeScenarioDir({ - nodeArgs: ["--import", "braintrust/hook.mjs"], - scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - - const contract = assertAnthropicTraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "anthropic-auto-hook-root", - scenarioName: "anthropic-auto-instrumentation-node-hook", - }); - - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }, - ); - }, -); diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json new file mode 100644 index 000000000..2ab51b170 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.log-payloads.json @@ -0,0 +1,305 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json new file mode 100644 index 000000000..187e089f8 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0273.span-events.json @@ -0,0 +1,243 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metric_keys": [], + "name": "anthropic-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "create" + }, + "metric_keys": [], + "name": "anthropic-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "anthropic-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "anthropic-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-with-response" + }, + "metric_keys": [], + "name": "anthropic-stream-with-response-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-tool" + }, + "metric_keys": [], + "name": "anthropic-stream-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "anthropic-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json new file mode 100644 index 000000000..30d2b4d04 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.log-payloads.json @@ -0,0 +1,389 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 6, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "start": 0, + "time_to_first_token": 0, + "tokens": 19 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "BETA.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + } +] diff --git a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json similarity index 86% rename from e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json rename to e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json index 7d054c870..0bfc89b4c 100644 --- a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0390.span-events.json @@ -3,10 +3,10 @@ "has_input": false, "has_output": false, "metadata": { - "scenario": "wrap-anthropic-message-traces" + "scenario": "anthropic-instrumentation" }, "metric_keys": [], - "name": "anthropic-wrapper-root", + "name": "anthropic-instrumentation-root", "root_span_id": "", "span_id": "", "span_parents": [], @@ -168,10 +168,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "tool" + "operation": "stream-tool" }, "metric_keys": [], - "name": "anthropic-tool-operation", + "name": "anthropic-stream-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -206,10 +206,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-create" + "operation": "tool" }, "metric_keys": [], - "name": "anthropic-beta-create-operation", + "name": "anthropic-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -244,10 +244,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-stream" + "operation": "beta-create" }, "metric_keys": [], - "name": "anthropic-beta-stream-operation", + "name": "anthropic-beta-create-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -277,5 +277,43 @@ "" ], "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" } ] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json new file mode 100644 index 000000000..30d2b4d04 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.log-payloads.json @@ -0,0 +1,389 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 6, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "start": 0, + "time_to_first_token": 0, + "tokens": 19 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "BETA.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json similarity index 85% rename from e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json rename to e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json index 6b468d32b..0bfc89b4c 100644 --- a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/__snapshots__/span-events.json +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0712.span-events.json @@ -3,10 +3,10 @@ "has_input": false, "has_output": false, "metadata": { - "scenario": "anthropic-auto-instrumentation-node-hook" + "scenario": "anthropic-instrumentation" }, "metric_keys": [], - "name": "anthropic-auto-hook-root", + "name": "anthropic-instrumentation-root", "root_span_id": "", "span_id": "", "span_parents": [], @@ -168,10 +168,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "tool" + "operation": "stream-tool" }, "metric_keys": [], - "name": "anthropic-tool-operation", + "name": "anthropic-stream-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -206,10 +206,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-create" + "operation": "tool" }, "metric_keys": [], - "name": "anthropic-beta-create-operation", + "name": "anthropic-tool-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -232,7 +232,7 @@ "time_to_first_token", "tokens" ], - "name": "anthropic.beta.messages.create", + "name": "anthropic.messages.create", "root_span_id": "", "span_id": "", "span_parents": [ @@ -244,10 +244,10 @@ "has_input": false, "has_output": false, "metadata": { - "operation": "beta-stream" + "operation": "beta-create" }, "metric_keys": [], - "name": "anthropic-beta-stream-operation", + "name": "anthropic-beta-create-operation", "root_span_id": "", "span_id": "", "span_parents": [ @@ -270,12 +270,50 @@ "time_to_first_token", "tokens" ], - "name": "anthropic.beta.messages.create", + "name": "anthropic.messages.create", "root_span_id": "", "span_id": "", "span_parents": [ "" ], "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" } ] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json new file mode 100644 index 000000000..30d2b4d04 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.log-payloads.json @@ -0,0 +1,389 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 6, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "start": 0, + "time_to_first_token": 0, + "tokens": 19 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "BETA.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json new file mode 100644 index 000000000..0bfc89b4c --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0730.span-events.json @@ -0,0 +1,319 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metric_keys": [], + "name": "anthropic-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "create" + }, + "metric_keys": [], + "name": "anthropic-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "anthropic-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "anthropic-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-with-response" + }, + "metric_keys": [], + "name": "anthropic-stream-with-response-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-tool" + }, + "metric_keys": [], + "name": "anthropic-stream-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "anthropic-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-create" + }, + "metric_keys": [], + "name": "anthropic-beta-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json new file mode 100644 index 000000000..30d2b4d04 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.log-payloads.json @@ -0,0 +1,389 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 6, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "start": 0, + "time_to_first_token": 0, + "tokens": 19 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "BETA.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json new file mode 100644 index 000000000..0bfc89b4c --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0780.span-events.json @@ -0,0 +1,319 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metric_keys": [], + "name": "anthropic-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "create" + }, + "metric_keys": [], + "name": "anthropic-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "anthropic-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "anthropic-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-with-response" + }, + "metric_keys": [], + "name": "anthropic-stream-with-response-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-tool" + }, + "metric_keys": [], + "name": "anthropic-stream-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "anthropic-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-create" + }, + "metric_keys": [], + "name": "anthropic-beta-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json new file mode 100644 index 000000000..30d2b4d04 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.log-payloads.json @@ -0,0 +1,389 @@ +[ + { + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-instrumentation-root", + "type": "task" + }, + { + "metadata": { + "operation": "create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly OK.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 5, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 12, + "start": 0, + "time_to_first_token": 0, + "tokens": 17 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "OK.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "attachment" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-attachment-operation", + "type": null + }, + { + "input": [ + { + "content": [ + { + "text": "Describe the attached image in one short sentence.", + "type": "text" + }, + { + "source": { + "data": { + "content_type": "image/png", + "filename": "image.png", + "key": "", + "type": "braintrust_attachment" + }, + "media_type": "image/png", + "type": "base64" + }, + "type": "image" + } + ], + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 29, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 1389, + "start": 0, + "time_to_first_token": 0, + "tokens": 1418 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-with-response" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-with-response-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + }, + { + "metadata": { + "operation": "stream-tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-stream-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 26, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 454, + "start": 0, + "time_to_first_token": 0, + "tokens": 480 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "tool" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-tool-operation", + "type": null + }, + { + "input": [ + { + "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "tool_use", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 55, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 357, + "start": 0, + "time_to_first_token": 0, + "tokens": 412 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "caller": { + "type": "direct" + }, + "id": "", + "input": { + "location": "Paris, France" + }, + "name": "get_weather", + "type": "tool_use" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-create" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-create-operation", + "type": null + }, + { + "input": [ + { + "content": "Reply with exactly BETA.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 6, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 13, + "start": 0, + "time_to_first_token": 0, + "tokens": 19 + }, + "name": "anthropic.messages.create", + "output": { + "content": [ + { + "text": "BETA.", + "type": "text" + } + ], + "role": "assistant" + }, + "type": "llm" + }, + { + "metadata": { + "operation": "beta-stream" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "name": "anthropic-beta-stream-operation", + "type": null + }, + { + "input": [ + { + "content": "Count from 1 to 3 and include the words one two three.", + "role": "user" + } + ], + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic", + "stop_reason": "end_turn", + "stop_sequence": null + }, + "metrics": { + "completion_tokens": 18, + "end": 0, + "prompt_cache_creation_tokens": 0, + "prompt_cached_tokens": 0, + "prompt_tokens": 24, + "start": 0, + "time_to_first_token": 0, + "tokens": 42 + }, + "name": "anthropic.messages.create", + "output": "1 - one\n2 - two\n3 - three", + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json new file mode 100644 index 000000000..0bfc89b4c --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/__snapshots__/anthropic-v0800.span-events.json @@ -0,0 +1,319 @@ +[ + { + "has_input": false, + "has_output": false, + "metadata": { + "scenario": "anthropic-instrumentation" + }, + "metric_keys": [], + "name": "anthropic-instrumentation-root", + "root_span_id": "", + "span_id": "", + "span_parents": [], + "type": "task" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "create" + }, + "metric_keys": [], + "name": "anthropic-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "attachment" + }, + "metric_keys": [], + "name": "anthropic-attachment-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream" + }, + "metric_keys": [], + "name": "anthropic-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-with-response" + }, + "metric_keys": [], + "name": "anthropic-stream-with-response-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "stream-tool" + }, + "metric_keys": [], + "name": "anthropic-stream-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "tool" + }, + "metric_keys": [], + "name": "anthropic-tool-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-create" + }, + "metric_keys": [], + "name": "anthropic-beta-create-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + }, + { + "has_input": false, + "has_output": false, + "metadata": { + "operation": "beta-stream" + }, + "metric_keys": [], + "name": "anthropic-beta-stream-operation", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": null + }, + { + "has_input": true, + "has_output": true, + "metadata": { + "model": "claude-3-haiku-20240307", + "provider": "anthropic" + }, + "metric_keys": [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "time_to_first_token", + "tokens" + ], + "name": "anthropic.messages.create", + "root_span_id": "", + "span_id": "", + "span_parents": [ + "" + ], + "type": "llm" + } +] diff --git a/e2e/scenarios/anthropic-instrumentation/assertions.ts b/e2e/scenarios/anthropic-instrumentation/assertions.ts new file mode 100644 index 000000000..b30f9787d --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/assertions.ts @@ -0,0 +1,527 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import { normalizeForSnapshot, type Json } from "../../helpers/normalize"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import { withScenarioHarness } from "../../helpers/scenario-harness"; +import { findChildSpans, findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeWrapperContract } from "../../helpers/wrapper-contract"; +import { E2E_TAGS } from "../../helpers/tags"; +import { ROOT_NAME, SCENARIO_NAME } from "./scenario.impl.mjs"; + +type RunAnthropicScenario = (harness: { + runNodeScenarioDir: (options: { + nodeArgs: string[]; + scenarioDir: string; + timeoutMs: number; + }) => Promise; + runScenarioDir: (options: { + scenarioDir: string; + timeoutMs: number; + }) => Promise; +}) => Promise; + +function findAnthropicSpan( + events: CapturedLogEvent[], + parentId: string | undefined, + names: string[], +) { + for (const name of names) { + const span = findChildSpans(events, name, parentId)[0]; + if (span) { + return span; + } + } + + return undefined; +} + +function pickMetadata( + metadata: Record | undefined, + keys: string[], +): Json { + if (!metadata) { + return null; + } + + const picked = Object.fromEntries( + keys.flatMap((key) => + key in metadata ? [[key, metadata[key] as Json]] : [], + ), + ); + + return Object.keys(picked).length > 0 ? (picked as Json) : null; +} + +function summarizeAnthropicPayload(event: CapturedLogEvent): Json { + const summary = { + input: event.input as Json, + metadata: pickMetadata( + event.row.metadata as Record | undefined, + [ + "provider", + "model", + "operation", + "scenario", + "stop_reason", + "stop_sequence", + ], + ), + metrics: event.metrics as Json, + name: event.span.name ?? null, + output: event.output as Json, + type: event.span.type ?? null, + } satisfies Json; + + if ( + event.span.name !== "anthropic.messages.create" || + !Array.isArray((summary.output as { content?: unknown[] } | null)?.content) + ) { + return summary; + } + + const output = structuredClone( + summary.output as { + content: Array<{ text?: string; type?: string }>; + }, + ); + const textBlock = output.content.find( + (block) => block.type === "text" && typeof block.text === "string", + ); + const input = event.input as + | Array<{ + content?: + | string + | Array<{ + source?: { + data?: { + type?: string; + }; + }; + }>; + }> + | undefined; + const hasAttachmentInput = input?.some( + (message) => + Array.isArray(message.content) && + message.content.some( + (block) => block.source?.data?.type === "braintrust_attachment", + ), + ); + + if (hasAttachmentInput && textBlock) { + textBlock.text = ""; + summary.output = output as Json; + } + + return summary; +} + +function buildSpanSummary( + events: CapturedLogEvent[], + supportsBetaMessages: boolean, +): Json { + const createOperation = findLatestSpan(events, "anthropic-create-operation"); + const attachmentOperation = findLatestSpan( + events, + "anthropic-attachment-operation", + ); + const streamOperation = findLatestSpan(events, "anthropic-stream-operation"); + const withResponseOperation = findLatestSpan( + events, + "anthropic-stream-with-response-operation", + ); + const toolStreamOperation = findLatestSpan( + events, + "anthropic-stream-tool-operation", + ); + const toolOperation = findLatestSpan(events, "anthropic-tool-operation"); + const betaCreateOperation = findLatestSpan( + events, + "anthropic-beta-create-operation", + ); + const betaStreamOperation = findLatestSpan( + events, + "anthropic-beta-stream-operation", + ); + + return normalizeForSnapshot( + [ + findLatestSpan(events, ROOT_NAME), + createOperation, + findAnthropicSpan(events, createOperation?.span.id, [ + "anthropic.messages.create", + ]), + attachmentOperation, + findAnthropicSpan(events, attachmentOperation?.span.id, [ + "anthropic.messages.create", + ]), + streamOperation, + findAnthropicSpan(events, streamOperation?.span.id, [ + "anthropic.messages.create", + ]), + withResponseOperation, + findAnthropicSpan(events, withResponseOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolStreamOperation, + findAnthropicSpan(events, toolStreamOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolOperation, + findAnthropicSpan(events, toolOperation?.span.id, [ + "anthropic.messages.create", + ]), + ...(supportsBetaMessages + ? [ + betaCreateOperation, + findAnthropicSpan(events, betaCreateOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + betaStreamOperation, + findAnthropicSpan(events, betaStreamOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + ] + : []), + ].map((event) => + summarizeWrapperContract(event!, [ + "provider", + "model", + "operation", + "scenario", + ]), + ) as Json, + ); +} + +function buildPayloadSummary( + events: CapturedLogEvent[], + supportsBetaMessages: boolean, +): Json { + const createOperation = findLatestSpan(events, "anthropic-create-operation"); + const attachmentOperation = findLatestSpan( + events, + "anthropic-attachment-operation", + ); + const streamOperation = findLatestSpan(events, "anthropic-stream-operation"); + const withResponseOperation = findLatestSpan( + events, + "anthropic-stream-with-response-operation", + ); + const toolStreamOperation = findLatestSpan( + events, + "anthropic-stream-tool-operation", + ); + const toolOperation = findLatestSpan(events, "anthropic-tool-operation"); + const betaCreateOperation = findLatestSpan( + events, + "anthropic-beta-create-operation", + ); + const betaStreamOperation = findLatestSpan( + events, + "anthropic-beta-stream-operation", + ); + + return normalizeForSnapshot( + [ + findLatestSpan(events, ROOT_NAME), + createOperation, + findAnthropicSpan(events, createOperation?.span.id, [ + "anthropic.messages.create", + ]), + attachmentOperation, + findAnthropicSpan(events, attachmentOperation?.span.id, [ + "anthropic.messages.create", + ]), + streamOperation, + findAnthropicSpan(events, streamOperation?.span.id, [ + "anthropic.messages.create", + ]), + withResponseOperation, + findAnthropicSpan(events, withResponseOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolStreamOperation, + findAnthropicSpan(events, toolStreamOperation?.span.id, [ + "anthropic.messages.create", + ]), + toolOperation, + findAnthropicSpan(events, toolOperation?.span.id, [ + "anthropic.messages.create", + ]), + ...(supportsBetaMessages + ? [ + betaCreateOperation, + findAnthropicSpan(events, betaCreateOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + betaStreamOperation, + findAnthropicSpan(events, betaStreamOperation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]), + ] + : []), + ].map((event) => summarizeAnthropicPayload(event!)) as Json, + ); +} + +export function defineAnthropicInstrumentationAssertions(options: { + name: string; + snapshotName: string; + supportsBetaMessages: boolean; + testFileUrl: string; + timeoutMs: number; + runScenario: RunAnthropicScenario; +}): void { + const spanSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.span-events.json`, + ); + const payloadSnapshotPath = resolveFileSnapshotPath( + options.testFileUrl, + `${options.snapshotName}.log-payloads.json`, + ); + const testConfig = { + tags: [E2E_TAGS.externalApi], + timeout: options.timeoutMs, + }; + + describe(options.name, () => { + let events: CapturedLogEvent[] = []; + + beforeAll(async () => { + await withScenarioHarness(async (harness) => { + await options.runScenario(harness); + events = harness.events(); + }); + }, options.timeoutMs); + + test("captures the root trace for the scenario", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + + expect(root).toBeDefined(); + expect(root?.row.metadata).toMatchObject({ + scenario: SCENARIO_NAME, + }); + }); + + test("captures trace for client.messages.create()", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "anthropic-create-operation"); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect( + typeof (span?.row.metadata as { model?: unknown } | undefined)?.model, + ).toBe("string"); + }); + + test("captures trace for sending an attachment", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-attachment-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(JSON.stringify(span?.input)).toContain("image.png"); + }); + + test( + "captures trace for client.messages.create({ stream: true })", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "anthropic-stream-operation"); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + }, + ); + + test("captures trace for the second streaming path", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-stream-with-response-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + }); + + test("captures trace for streamed tool use", testConfig, () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-stream-tool-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + const output = span?.output as + | { content?: Array<{ name?: string; type?: string }> } + | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + expect( + output?.content?.some( + (block) => block.type === "tool_use" && block.name === "get_weather", + ), + ).toBe(true); + }); + + test( + "captures trace for client.messages.create() with tools", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan(events, "anthropic-tool-operation"); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + ]); + const output = span?.output as + | { content?: Array<{ name?: string; type?: string }> } + | undefined; + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect( + output?.content?.some( + (block) => + block.type === "tool_use" && block.name === "get_weather", + ), + ).toBe(true); + }, + ); + + if (options.supportsBetaMessages) { + test( + "captures trace for client.beta.messages.create()", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-beta-create-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + }, + ); + + test( + "captures trace for client.beta.messages.create({ stream: true })", + testConfig, + () => { + const root = findLatestSpan(events, ROOT_NAME); + const operation = findLatestSpan( + events, + "anthropic-beta-stream-operation", + ); + const span = findAnthropicSpan(events, operation?.span.id, [ + "anthropic.messages.create", + "anthropic.beta.messages.create", + ]); + + expect(operation).toBeDefined(); + expect(span).toBeDefined(); + expect(operation?.span.parentIds).toEqual([root?.span.id ?? ""]); + expect(span?.row.metadata).toMatchObject({ + provider: "anthropic", + }); + expect(span?.metrics).toMatchObject({ + time_to_first_token: expect.any(Number), + prompt_tokens: expect.any(Number), + completion_tokens: expect.any(Number), + }); + }, + ); + } + + test("matches the shared span snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot( + buildSpanSummary(events, options.supportsBetaMessages), + ), + ).toMatchFileSnapshot(spanSnapshotPath); + }); + + test("matches the shared payload snapshot", testConfig, async () => { + await expect( + formatJsonFileSnapshot( + buildPayloadSummary(events, options.supportsBetaMessages), + ), + ).toMatchFileSnapshot(payloadSnapshotPath); + }); + }); +} diff --git a/e2e/scenarios/anthropic-instrumentation/package.json b/e2e/scenarios/anthropic-instrumentation/package.json new file mode 100644 index 000000000..47b6c69f6 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/package.json @@ -0,0 +1,19 @@ +{ + "name": "@braintrust/e2e-anthropic-instrumentation", + "private": true, + "braintrustScenario": { + "canary": { + "dependencies": { + "@anthropic-ai/sdk": "latest" + } + } + }, + "dependencies": { + "@anthropic-ai/sdk": "0.80.0", + "anthropic-sdk-v0273": "npm:@anthropic-ai/sdk@0.27.3", + "anthropic-sdk-v0390": "npm:@anthropic-ai/sdk@0.39.0", + "anthropic-sdk-v0712": "npm:@anthropic-ai/sdk@0.71.2", + "anthropic-sdk-v0730": "npm:@anthropic-ai/sdk@0.73.0", + "anthropic-sdk-v0780": "npm:@anthropic-ai/sdk@0.78.0" + } +} diff --git a/e2e/scenarios/anthropic-instrumentation/pnpm-lock.yaml b/e2e/scenarios/anthropic-instrumentation/pnpm-lock.yaml new file mode 100644 index 000000000..f591c2761 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/pnpm-lock.yaml @@ -0,0 +1,406 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@anthropic-ai/sdk': + specifier: 0.80.0 + version: 0.80.0 + anthropic-sdk-v0273: + specifier: npm:@anthropic-ai/sdk@0.27.3 + version: '@anthropic-ai/sdk@0.27.3' + anthropic-sdk-v0390: + specifier: npm:@anthropic-ai/sdk@0.39.0 + version: '@anthropic-ai/sdk@0.39.0' + anthropic-sdk-v0712: + specifier: npm:@anthropic-ai/sdk@0.71.2 + version: '@anthropic-ai/sdk@0.71.2' + anthropic-sdk-v0730: + specifier: npm:@anthropic-ai/sdk@0.73.0 + version: '@anthropic-ai/sdk@0.73.0' + anthropic-sdk-v0780: + specifier: npm:@anthropic-ai/sdk@0.78.0 + version: '@anthropic-ai/sdk@0.78.0' + +packages: + + '@anthropic-ai/sdk@0.27.3': + resolution: {integrity: sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw==} + + '@anthropic-ai/sdk@0.39.0': + resolution: {integrity: sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg==} + + '@anthropic-ai/sdk@0.71.2': + resolution: {integrity: sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@anthropic-ai/sdk@0.73.0': + resolution: {integrity: sha512-URURVzhxXGJDGUGFunIOtBlSl7KWvZiAAKY/ttTkZAkXT9bTPqdk2eK0b8qqSxXpikh3QKPnPYpiyX98zf5ebw==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@anthropic-ai/sdk@0.78.0': + resolution: {integrity: sha512-PzQhR715td/m1UaaN5hHXjYB8Gl2lF9UVhrrGrZeysiF6Rb74Wc9GCB8hzLdzmQtBd1qe89F9OptgB9Za1Ib5w==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@anthropic-ai/sdk@0.80.0': + resolution: {integrity: sha512-WeXLn7zNVk3yjeshn+xZHvld6AoFUOR3Sep6pSoHho5YbSi6HwcirqgPA5ccFuW8QTVJAAU7N8uQQC6Wa9TG+g==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@babel/runtime@7.29.2': + resolution: {integrity: sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g==} + engines: {node: '>=6.9.0'} + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.5: + resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + +snapshots: + + '@anthropic-ai/sdk@0.27.3': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + '@anthropic-ai/sdk@0.39.0': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + '@anthropic-ai/sdk@0.71.2': + dependencies: + json-schema-to-ts: 3.1.1 + + '@anthropic-ai/sdk@0.73.0': + dependencies: + json-schema-to-ts: 3.1.1 + + '@anthropic-ai/sdk@0.78.0': + dependencies: + json-schema-to-ts: 3.1.1 + + '@anthropic-ai/sdk@0.80.0': + dependencies: + json-schema-to-ts: 3.1.1 + + '@babel/runtime@7.29.2': {} + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 18.19.130 + form-data: 4.0.5 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + asynckit@0.4.0: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + + delayed-stream@1.0.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + + event-target-shim@5.0.1: {} + + form-data-encoder@1.7.2: {} + + form-data@4.0.5: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.2 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + function-bind@1.1.2: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + gopd@1.2.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.29.2 + ts-algebra: 2.0.0 + + math-intrinsics@1.1.0: {} + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + ms@2.1.3: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + tr46@0.0.3: {} + + ts-algebra@2.0.0: {} + + undici-types@5.26.5: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs new file mode 100644 index 000000000..f33a4f984 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.mjs @@ -0,0 +1,7 @@ +import Anthropic from "anthropic-sdk-v0273"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => + runAutoAnthropicInstrumentation(Anthropic, { useBetaMessages: false }), +); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts new file mode 100644 index 000000000..f6d3267ec --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0273.ts @@ -0,0 +1,7 @@ +import Anthropic from "anthropic-sdk-v0273"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => + runWrappedAnthropicInstrumentation(Anthropic, { useBetaMessages: false }), +); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs new file mode 100644 index 000000000..0fc19dd85 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.mjs @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0390"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts new file mode 100644 index 000000000..47743faae --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0390.ts @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0390"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs new file mode 100644 index 000000000..1ae3d826e --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.mjs @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0712"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts new file mode 100644 index 000000000..ba40a6f24 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0712.ts @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0712"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs new file mode 100644 index 000000000..3485fb6c6 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.mjs @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0730"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts new file mode 100644 index 000000000..d7d4cf5f4 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0730.ts @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0730"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs new file mode 100644 index 000000000..f7e839fec --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.mjs @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0780"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts new file mode 100644 index 000000000..de5f645c2 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.anthropic-v0780.ts @@ -0,0 +1,5 @@ +import Anthropic from "anthropic-sdk-v0780"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); diff --git a/e2e/helpers/anthropic-scenario.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs similarity index 56% rename from e2e/helpers/anthropic-scenario.mjs rename to e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs index 80aa4ebf3..64f803352 100644 --- a/e2e/helpers/anthropic-scenario.mjs +++ b/e2e/scenarios/anthropic-instrumentation/scenario.impl.mjs @@ -1,20 +1,44 @@ import { readFile } from "node:fs/promises"; +import { wrapAnthropic } from "braintrust"; import { collectAsync, runOperation, runTracedScenario, -} from "./provider-runtime.mjs"; +} from "../../helpers/provider-runtime.mjs"; const ANTHROPIC_MODEL = "claude-3-haiku-20240307"; +const ROOT_NAME = "anthropic-instrumentation-root"; +const SCENARIO_NAME = "anthropic-instrumentation"; +const WEATHER_TOOL = { + name: "get_weather", + description: "Get the current weather in a given location", + input_schema: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state or city and country", + }, + }, + required: ["location"], + }, +}; -export async function runAnthropicScenario(options) { - const imageBase64 = (await readFile(options.testImageUrl)).toString("base64"); - const baseClient = new options.Anthropic({ +async function runAnthropicInstrumentationScenario( + Anthropic, + { + decorateClient, + useBetaMessages = true, + useMessagesStreamHelper = true, + } = {}, +) { + const imageBase64 = ( + await readFile(new URL("./test-image.png", import.meta.url)) + ).toString("base64"); + const baseClient = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY, }); - const client = options.decorateClient - ? options.decorateClient(baseClient) - : baseClient; + const client = decorateClient ? decorateClient(baseClient) : baseClient; await runTracedScenario({ callback: async () => { @@ -79,7 +103,7 @@ export async function runAnthropicScenario(options) { "stream-with-response", async () => { const stream = - options.useMessagesStreamHelper === false + useMessagesStreamHelper === false ? await client.messages.create({ model: ANTHROPIC_MODEL, max_tokens: 32, @@ -109,27 +133,39 @@ export async function runAnthropicScenario(options) { }, ); + await runOperation( + "anthropic-stream-tool-operation", + "stream-tool", + async () => { + const stream = await client.messages.create({ + model: ANTHROPIC_MODEL, + max_tokens: 128, + temperature: 0, + stream: true, + tool_choice: { + type: "tool", + name: WEATHER_TOOL.name, + disable_parallel_tool_use: true, + }, + tools: [WEATHER_TOOL], + messages: [ + { + role: "user", + content: + "Use the get_weather tool for Paris, France. Do not answer from memory.", + }, + ], + }); + await collectAsync(stream); + }, + ); + await runOperation("anthropic-tool-operation", "tool", async () => { await client.messages.create({ model: ANTHROPIC_MODEL, max_tokens: 128, temperature: 0, - tools: [ - { - name: "get_weather", - description: "Get the current weather in a given location", - input_schema: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state or city and country", - }, - }, - required: ["location"], - }, - }, - ], + tools: [WEATHER_TOOL], messages: [ { role: "user", @@ -140,44 +176,62 @@ export async function runAnthropicScenario(options) { }); }); - await runOperation( - "anthropic-beta-create-operation", - "beta-create", - async () => { - await client.beta.messages.create({ - model: ANTHROPIC_MODEL, - max_tokens: 16, - temperature: 0, - messages: [{ role: "user", content: "Reply with exactly BETA." }], - }); - }, - ); + if (useBetaMessages) { + await runOperation( + "anthropic-beta-create-operation", + "beta-create", + async () => { + await client.beta.messages.create({ + model: ANTHROPIC_MODEL, + max_tokens: 16, + temperature: 0, + messages: [{ role: "user", content: "Reply with exactly BETA." }], + }); + }, + ); - await runOperation( - "anthropic-beta-stream-operation", - "beta-stream", - async () => { - const stream = await client.beta.messages.create({ - model: ANTHROPIC_MODEL, - max_tokens: 32, - temperature: 0, - stream: true, - messages: [ - { - role: "user", - content: - "Count from 1 to 3 and include the words one two three.", - }, - ], - }); - await collectAsync(stream); - }, - ); + await runOperation( + "anthropic-beta-stream-operation", + "beta-stream", + async () => { + const stream = await client.beta.messages.create({ + model: ANTHROPIC_MODEL, + max_tokens: 32, + temperature: 0, + stream: true, + messages: [ + { + role: "user", + content: + "Count from 1 to 3 and include the words one two three.", + }, + ], + }); + await collectAsync(stream); + }, + ); + } }, metadata: { - scenario: options.scenarioName, + scenario: SCENARIO_NAME, }, - projectNameBase: options.projectNameBase, - rootName: options.rootName, + projectNameBase: "e2e-anthropic-instrumentation", + rootName: ROOT_NAME, }); } + +export async function runWrappedAnthropicInstrumentation(Anthropic, options) { + await runAnthropicInstrumentationScenario(Anthropic, { + decorateClient: wrapAnthropic, + ...options, + }); +} + +export async function runAutoAnthropicInstrumentation(Anthropic, options) { + await runAnthropicInstrumentationScenario(Anthropic, { + ...options, + useMessagesStreamHelper: false, + }); +} + +export { ROOT_NAME, SCENARIO_NAME }; diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.mjs b/e2e/scenarios/anthropic-instrumentation/scenario.mjs new file mode 100644 index 000000000..3c1844702 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.mjs @@ -0,0 +1,5 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { runMain } from "../../helpers/provider-runtime.mjs"; +import { runAutoAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runAutoAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.test.ts b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts new file mode 100644 index 000000000..baebbc01d --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.test.ts @@ -0,0 +1,99 @@ +import { describe } from "vitest"; +import { + prepareScenarioDir, + readInstalledPackageVersion, + resolveScenarioDir, +} from "../../helpers/scenario-harness"; +import { defineAnthropicInstrumentationAssertions } from "./assertions"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); +const TIMEOUT_MS = 90_000; +const anthropicScenarios = await Promise.all( + [ + { + autoEntry: "scenario.anthropic-v0273.mjs", + dependencyName: "anthropic-sdk-v0273", + snapshotName: "anthropic-v0273", + supportsBetaMessages: false, + wrapperEntry: "scenario.anthropic-v0273.ts", + }, + { + autoEntry: "scenario.anthropic-v0390.mjs", + dependencyName: "anthropic-sdk-v0390", + snapshotName: "anthropic-v0390", + supportsBetaMessages: true, + wrapperEntry: "scenario.anthropic-v0390.ts", + }, + { + autoEntry: "scenario.anthropic-v0712.mjs", + dependencyName: "anthropic-sdk-v0712", + snapshotName: "anthropic-v0712", + supportsBetaMessages: true, + wrapperEntry: "scenario.anthropic-v0712.ts", + }, + { + autoEntry: "scenario.anthropic-v0730.mjs", + dependencyName: "anthropic-sdk-v0730", + snapshotName: "anthropic-v0730", + supportsBetaMessages: true, + wrapperEntry: "scenario.anthropic-v0730.ts", + }, + { + autoEntry: "scenario.anthropic-v0780.mjs", + dependencyName: "anthropic-sdk-v0780", + snapshotName: "anthropic-v0780", + supportsBetaMessages: true, + wrapperEntry: "scenario.anthropic-v0780.ts", + }, + { + autoEntry: "scenario.mjs", + dependencyName: "@anthropic-ai/sdk", + snapshotName: "anthropic-v0800", + supportsBetaMessages: true, + wrapperEntry: "scenario.ts", + }, + ].map(async (scenario) => ({ + ...scenario, + version: await readInstalledPackageVersion( + scenarioDir, + scenario.dependencyName, + ), + })), +); + +for (const scenario of anthropicScenarios) { + describe(`anthropic sdk ${scenario.version}`, () => { + defineAnthropicInstrumentationAssertions({ + name: "wrapped instrumentation", + runScenario: async ({ runScenarioDir }) => { + await runScenarioDir({ + entry: scenario.wrapperEntry, + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + supportsBetaMessages: scenario.supportsBetaMessages, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + + defineAnthropicInstrumentationAssertions({ + name: "auto-hook instrumentation", + runScenario: async ({ runNodeScenarioDir }) => { + await runNodeScenarioDir({ + entry: scenario.autoEntry, + nodeArgs: ["--import", "braintrust/hook.mjs"], + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + }, + snapshotName: scenario.snapshotName, + supportsBetaMessages: scenario.supportsBetaMessages, + testFileUrl: import.meta.url, + timeoutMs: TIMEOUT_MS, + }); + }); +} diff --git a/e2e/scenarios/anthropic-instrumentation/scenario.ts b/e2e/scenarios/anthropic-instrumentation/scenario.ts new file mode 100644 index 000000000..5b5e688a2 --- /dev/null +++ b/e2e/scenarios/anthropic-instrumentation/scenario.ts @@ -0,0 +1,5 @@ +import Anthropic from "@anthropic-ai/sdk"; +import { runMain } from "../../helpers/scenario-runtime"; +import { runWrappedAnthropicInstrumentation } from "./scenario.impl.mjs"; + +runMain(async () => runWrappedAnthropicInstrumentation(Anthropic)); diff --git a/e2e/scenarios/anthropic-auto-instrumentation-node-hook/test-image.png b/e2e/scenarios/anthropic-instrumentation/test-image.png similarity index 100% rename from e2e/scenarios/anthropic-auto-instrumentation-node-hook/test-image.png rename to e2e/scenarios/anthropic-instrumentation/test-image.png diff --git a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json b/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json deleted file mode 100644 index 7e444e02e..000000000 --- a/e2e/scenarios/wrap-anthropic-message-traces/__snapshots__/log-payloads.json +++ /dev/null @@ -1,1154 +0,0 @@ -[ - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runTracedScenario", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "scenario": "wrap-anthropic-message-traces", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 0, - "name": "anthropic-wrapper-root", - "type": "task" - }, - "span_id": "" - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "" - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 1, - "name": "anthropic-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly OK.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 2, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 5, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 12, - "time_to_first_token": 0, - "tokens": 17 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "OK.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "attachment", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 3, - "name": "anthropic-attachment-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": [ - { - "text": "Describe the attached image in one short sentence.", - "type": "text" - }, - { - "source": { - "data": { - "content_type": "image/png", - "filename": "image.png", - "key": "", - "type": "braintrust_attachment" - }, - "media_type": "image/png", - "type": "base64" - }, - "type": "image" - } - ], - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 4, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 29, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1389, - "time_to_first_token": 0, - "tokens": 1418 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 5, - "name": "anthropic-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 6, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": null, - "stop_sequence": null - }, - "output": { - "content": [], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "stream-with-response", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 7, - "name": "anthropic-stream-with-response-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/scenarios/wrap-anthropic-message-traces/node_modules/.pnpm/@anthropic-ai+sdk@/node_modules/@anthropic-ai/sdk/lib/MessageStream.js", - "caller_functionname": "MessageStream._createMessage", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 8, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": null, - "stop_sequence": null - }, - "output": { - "content": [], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "tool", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 9, - "name": "anthropic-tool-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Use the get_weather tool for Paris, France. Do not answer from memory.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 128, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0, - "tools": [ - { - "description": "Get the current weather in a given location", - "input_schema": { - "properties": { - "location": { - "description": "The city and state or city and country", - "type": "string" - } - }, - "required": [ - "location" - ], - "type": "object" - }, - "name": "get_weather" - } - ] - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 10, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 55, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 357, - "time_to_first_token": 0, - "tokens": 412 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "tool_use", - "stop_sequence": null - }, - "output": { - "content": [ - { - "caller": { - "type": "direct" - }, - "id": "", - "input": { - "location": "Paris, France" - }, - "name": "get_weather", - "type": "tool_use" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-create", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 11, - "name": "anthropic-beta-create-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Reply with exactly BETA.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 16, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 12, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 6, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 13, - "time_to_first_token": 0, - "tokens": 19 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "output": { - "content": [ - { - "text": "BETA.", - "type": "text" - } - ], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "/e2e/helpers/provider-runtime.mjs", - "caller_functionname": "runOperation", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "log_id": "g", - "metadata": { - "operation": "beta-stream", - "testRunId": "" - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 13, - "name": "anthropic-beta-stream-operation" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": false, - "context": { - "caller_filename": "", - "caller_functionname": "", - "caller_lineno": 0 - }, - "created": "", - "id": "", - "input": [ - { - "content": "Count from 1 to 3 and include the words one two three.", - "role": "user" - } - ], - "log_id": "g", - "metadata": { - "max_tokens": 32, - "model": "claude-3-haiku-20240307", - "provider": "anthropic", - "stream": true, - "temperature": 0 - }, - "metrics": { - "start": 0 - }, - "project_id": "", - "root_span_id": "", - "span_attributes": { - "exec_counter": 14, - "name": "anthropic.messages.create", - "type": "llm" - }, - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 1, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "completion_tokens": 18, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 24, - "time_to_first_token": 0, - "tokens": 42 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": null, - "stop_sequence": null - }, - "output": { - "content": [], - "role": "assistant" - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "output": "1 - one\n2 - two\n3 - three", - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metadata": { - "stop_reason": "end_turn", - "stop_sequence": null - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - }, - { - "_is_merge": true, - "id": "", - "log_id": "g", - "metrics": { - "end": 0 - }, - "project_id": "", - "root_span_id": "", - "span_id": "", - "span_parents": [ - "" - ] - } -] diff --git a/e2e/scenarios/wrap-anthropic-message-traces/package.json b/e2e/scenarios/wrap-anthropic-message-traces/package.json deleted file mode 100644 index fb303580f..000000000 --- a/e2e/scenarios/wrap-anthropic-message-traces/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@braintrust/e2e-wrap-anthropic-message-traces", - "private": true, - "braintrustScenario": { - "canary": { - "dependencies": { - "@anthropic-ai/sdk": "latest" - } - } - }, - "dependencies": { - "@anthropic-ai/sdk": "0.60.0" - } -} diff --git a/e2e/scenarios/wrap-anthropic-message-traces/pnpm-lock.yaml b/e2e/scenarios/wrap-anthropic-message-traces/pnpm-lock.yaml deleted file mode 100644 index ea50cf7ab..000000000 --- a/e2e/scenarios/wrap-anthropic-message-traces/pnpm-lock.yaml +++ /dev/null @@ -1,23 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - '@anthropic-ai/sdk': - specifier: 0.60.0 - version: 0.60.0 - -packages: - - '@anthropic-ai/sdk@0.60.0': - resolution: {integrity: sha512-9zu/TXaUy8BZhXedDtt1wT3H4LOlpKDO1/ftiFpeR3N1PCr3KJFKkxxlQWWt1NNp08xSwUNJ3JNY8yhl8av6eQ==} - hasBin: true - -snapshots: - - '@anthropic-ai/sdk@0.60.0': {} diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts b/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts deleted file mode 100644 index a435c2d31..000000000 --- a/e2e/scenarios/wrap-anthropic-message-traces/scenario.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { expect, test } from "vitest"; -import { assertAnthropicTraceContract } from "../../helpers/anthropic-trace-contract"; -import { - formatJsonFileSnapshot, - resolveFileSnapshotPath, -} from "../../helpers/file-snapshot"; -import { - prepareScenarioDir, - resolveScenarioDir, - withScenarioHarness, -} from "../../helpers/scenario-harness"; -import { E2E_TAGS } from "../../helpers/tags"; - -const scenarioDir = await prepareScenarioDir({ - scenarioDir: resolveScenarioDir(import.meta.url), -}); -const TIMEOUT_MS = 90_000; - -test( - "wrap-anthropic-message-traces captures create, stream, beta, attachment, and tool spans", - { - tags: [E2E_TAGS.externalApi], - timeout: TIMEOUT_MS, - }, - async () => { - await withScenarioHarness(async ({ events, payloads, runScenarioDir }) => { - await runScenarioDir({ scenarioDir, timeoutMs: TIMEOUT_MS }); - - const contract = assertAnthropicTraceContract({ - capturedEvents: events(), - payloads: payloads(), - rootName: "anthropic-wrapper-root", - scenarioName: "wrap-anthropic-message-traces", - }); - - await expect( - formatJsonFileSnapshot(contract.spanSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "span-events.json"), - ); - await expect( - formatJsonFileSnapshot(contract.payloadSummary), - ).toMatchFileSnapshot( - resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), - ); - }); - }, -); diff --git a/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts b/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts deleted file mode 100644 index 95fa46e0f..000000000 --- a/e2e/scenarios/wrap-anthropic-message-traces/scenario.ts +++ /dev/null @@ -1,15 +0,0 @@ -import Anthropic from "@anthropic-ai/sdk"; -import { wrapAnthropic } from "braintrust"; -import { runAnthropicScenario } from "../../helpers/anthropic-scenario.mjs"; -import { runMain } from "../../helpers/scenario-runtime"; - -runMain(async () => - runAnthropicScenario({ - Anthropic, - decorateClient: wrapAnthropic, - projectNameBase: "e2e-wrap-anthropic", - rootName: "anthropic-wrapper-root", - scenarioName: "wrap-anthropic-message-traces", - testImageUrl: new URL("./test-image.png", import.meta.url), - }), -); diff --git a/e2e/scenarios/wrap-anthropic-message-traces/test-image.png b/e2e/scenarios/wrap-anthropic-message-traces/test-image.png deleted file mode 100644 index d59518eb3..000000000 Binary files a/e2e/scenarios/wrap-anthropic-message-traces/test-image.png and /dev/null differ diff --git a/js/src/auto-instrumentations/configs/anthropic.ts b/js/src/auto-instrumentations/configs/anthropic.ts index 940ffabae..048730471 100644 --- a/js/src/auto-instrumentations/configs/anthropic.ts +++ b/js/src/auto-instrumentations/configs/anthropic.ts @@ -13,12 +13,27 @@ import { anthropicChannels } from "../../instrumentation/plugins/anthropic-chann * "orchestrion:@anthropic-ai/sdk:messages.create" */ export const anthropicConfigs: InstrumentationConfig[] = [ + // Messages API - create in older SDK layouts (supports streaming via stream=true parameter) + { + channelName: anthropicChannels.messagesCreate.channelName, + module: { + name: "@anthropic-ai/sdk", + versionRange: ">=0.27.0 <0.39.0", + filePath: "resources/messages.mjs", + }, + functionQuery: { + className: "Messages", + methodName: "create", + kind: "Async", + }, + }, + // Messages API - create (supports streaming via stream=true parameter) { channelName: anthropicChannels.messagesCreate.channelName, module: { name: "@anthropic-ai/sdk", - versionRange: ">=0.60.0", + versionRange: ">=0.39.0", filePath: "resources/messages/messages.mjs", }, functionQuery: { @@ -33,7 +48,7 @@ export const anthropicConfigs: InstrumentationConfig[] = [ channelName: anthropicChannels.betaMessagesCreate.channelName, module: { name: "@anthropic-ai/sdk", - versionRange: ">=0.60.0", + versionRange: ">=0.39.0", filePath: "resources/beta/messages/messages.mjs", }, functionQuery: { diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/plugins/anthropic-plugin.test.ts index 19a8f2e58..889f83d08 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.test.ts @@ -385,6 +385,77 @@ describe("aggregateAnthropicStreamChunks", () => { expect(result.output).toBe("Hello world"); }); + it("should aggregate streamed tool_use output", () => { + const chunks = [ + { + type: "message_start", + message: { + role: "assistant", + usage: { + input_tokens: 25, + output_tokens: 0, + }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: "toolu_123", + name: "get_weather", + input: {}, + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "input_json_delta", + partial_json: '{"location":"Paris, France"}', + }, + }, + { + type: "content_block_stop", + index: 0, + }, + { + type: "message_delta", + delta: { + stop_reason: "tool_use", + stop_sequence: null, + }, + usage: { + output_tokens: 12, + }, + }, + ]; + + const result = aggregateAnthropicStreamChunks(chunks); + + expect(result.output).toEqual({ + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_123", + name: "get_weather", + input: { + location: "Paris, France", + }, + }, + ], + }); + expect(result.metrics).toMatchObject({ + prompt_tokens: 25, + completion_tokens: 12, + }); + expect(result.metadata).toEqual({ + stop_reason: "tool_use", + stop_sequence: null, + }); + }); + it("should handle chunks without delta or usage", () => { const chunks = [ { type: "message_start" }, diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/plugins/anthropic-plugin.ts index f0dbbf150..5bb621611 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/plugins/anthropic-plugin.ts @@ -10,6 +10,7 @@ import type { AnthropicCreateParams, AnthropicInputMessage, AnthropicMessage, + AnthropicOutputContentBlock, AnthropicStreamEvent, AnthropicUsage, } from "../../vendor-sdk-types/anthropic"; @@ -91,7 +92,7 @@ export class AnthropicPlugin extends BasePlugin { this.unsubscribers.push( traceStreamingChannel(anthropicChannels.betaMessagesCreate, { ...anthropicConfig, - name: "anthropic.beta.messages.create", + name: "anthropic.messages.create", }), ); } @@ -138,13 +139,16 @@ export function parseMetricsFromUsage( export function aggregateAnthropicStreamChunks( chunks: AnthropicStreamEvent[], ): { - output: string; + output: unknown; metrics: Record; metadata: Record; } { - const deltas: string[] = []; + const fallbackTextDeltas: string[] = []; + const contentBlocks: Record = {}; + const contentBlockDeltas: Record = {}; let metrics: Record = {}; let metadata: Record = {}; + let role: string | undefined; for (const event of chunks) { switch (event?.type) { @@ -154,18 +158,50 @@ export function aggregateAnthropicStreamChunks( const initialMetrics = parseMetricsFromUsage(event.message.usage); metrics = { ...metrics, ...initialMetrics }; } + if (typeof event.message?.role === "string") { + role = event.message.role; + } + break; + + case "content_block_start": + if (event.content_block) { + contentBlocks[event.index] = event.content_block; + contentBlockDeltas[event.index] = []; + } break; case "content_block_delta": - // Collect text deltas if (event.delta?.type === "text_delta") { const text = event.delta.text; if (text) { - deltas.push(text); + if ( + contentBlocks[event.index] !== undefined || + contentBlockDeltas[event.index] !== undefined + ) { + contentBlockDeltas[event.index] ??= []; + contentBlockDeltas[event.index].push(text); + } else { + fallbackTextDeltas.push(text); + } + } + } else if (event.delta?.type === "input_json_delta") { + const partialJson = event.delta.partial_json; + if (partialJson) { + contentBlockDeltas[event.index] ??= []; + contentBlockDeltas[event.index].push(partialJson); } } break; + case "content_block_stop": + finalizeContentBlock( + event.index, + contentBlocks, + contentBlockDeltas, + fallbackTextDeltas, + ); + break; + case "message_delta": // Collect final usage stats and metadata if (event.usage) { @@ -180,7 +216,26 @@ export function aggregateAnthropicStreamChunks( } } - const output = deltas.join(""); + const orderedContent = Object.entries(contentBlocks) + .map(([index, block]) => ({ + block, + index: Number(index), + })) + .filter(({ block }) => block !== undefined) + .sort((left, right) => left.index - right.index) + .map(({ block }) => block); + + let output: unknown = fallbackTextDeltas.join(""); + if (orderedContent.length > 0) { + if (orderedContent.every(isTextContentBlock)) { + output = orderedContent.map((block) => block.text).join(""); + } else { + output = { + ...(role ? { role } : {}), + content: orderedContent, + }; + } + } const finalized = finalizeAnthropicTokens(metrics); // Filter out undefined values to match Record type @@ -197,6 +252,67 @@ export function aggregateAnthropicStreamChunks( }; } +function finalizeContentBlock( + index: number, + contentBlocks: Record, + contentBlockDeltas: Record, + fallbackTextDeltas: string[], +): void { + const contentBlock = contentBlocks[index]; + if (!contentBlock) { + return; + } + + const text = contentBlockDeltas[index]?.join("") ?? ""; + + if (isToolUseContentBlock(contentBlock)) { + if (!text) { + return; + } + + try { + contentBlocks[index] = { + ...contentBlock, + input: JSON.parse(text), + }; + } catch { + fallbackTextDeltas.push(text); + delete contentBlocks[index]; + } + return; + } + + if (isTextContentBlock(contentBlock)) { + if (!text) { + delete contentBlocks[index]; + return; + } + + contentBlocks[index] = { + ...contentBlock, + text, + }; + return; + } + + if (text) { + fallbackTextDeltas.push(text); + } + delete contentBlocks[index]; +} + +function isTextContentBlock( + contentBlock: AnthropicOutputContentBlock, +): contentBlock is Extract { + return contentBlock.type === "text"; +} + +function isToolUseContentBlock( + contentBlock: AnthropicOutputContentBlock, +): contentBlock is Extract { + return contentBlock.type === "tool_use"; +} + function isAnthropicBase64ContentBlock( input: Record, ): input is Record & {