From 75170762bcdb703664ea399115e16a8a3049abe1 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 23 Mar 2026 10:48:19 -0700 Subject: [PATCH] refactor(e2e): migrate deno scenarios from smoke to e2e harness Add `runDenoScenarioDir()` to the e2e scenario harness so Deno tests can run alongside Node/tsx scenarios under the same mock-server infrastructure. Remove the standalone `deno-node` and `deno-browser` smoke scenarios and the legacy `js/smoke/tests/deno/deno.json` config, which are superseded by `e2e/scenarios/deno-node/` and `e2e/scenarios/deno-browser/`. - Refactor `runProcess` to accept an explicit `command` arg so it can spawn arbitrary executables (not just `process.execPath`) - Add `runDenoScenarioDir()` that invokes `deno test --no-check` with the standard harness env vars; defaults entry to `runner.case.ts` - Expose `runDenoScenarioDir` on the `ScenarioHarness` interface and wire it through `withScenarioHarness` - Update `e2e/README.md` and `.agents/skills/e2e-tests/SKILL.md` to document the new helper and Deno-specific conventions - Clean up `js/smoke/README.md` and `js/smoke/shared/README.md` to remove references to the deleted smoke scenarios --- .agents/skills/e2e-tests/SKILL.md | 15 +- e2e/README.md | 3 + e2e/helpers/deno-test-helpers.ts | 192 ++++++ e2e/helpers/scenario-harness.ts | 57 +- .../__snapshots__/log-payloads.json | 242 +++++++ .../__snapshots__/request-flow.json | 520 +++++++++++++++ .../__snapshots__/span-events.json | 177 +++++ e2e/scenarios/deno-browser/runner.case.ts | 208 ++++++ e2e/scenarios/deno-browser/scenario.test.ts | 191 ++++++ .../deno-node/__snapshots__/log-payloads.json | 286 ++++++++ .../deno-node/__snapshots__/request-flow.json | 608 ++++++++++++++++++ .../deno-node/__snapshots__/span-events.json | 184 ++++++ e2e/scenarios/deno-node/runner.case.ts | 205 ++++++ e2e/scenarios/deno-node/scenario.test.ts | 201 ++++++ js/smoke/README.md | 9 +- js/smoke/scenarios/deno-browser/.gitignore | 4 - js/smoke/scenarios/deno-browser/Makefile | 19 - js/smoke/scenarios/deno-browser/README.md | 35 - js/smoke/scenarios/deno-browser/deno.json | 9 - js/smoke/scenarios/deno-browser/mise.toml | 3 - .../deno-browser/tests/shared-suite.test.ts | 83 --- js/smoke/scenarios/deno-node/.gitignore | 4 - js/smoke/scenarios/deno-node/Makefile | 19 - js/smoke/scenarios/deno-node/README.md | 27 - js/smoke/scenarios/deno-node/deno.json | 9 - js/smoke/scenarios/deno-node/mise.toml | 3 - .../deno-node/tests/shared-suite.test.ts | 75 --- js/smoke/shared/README.md | 4 +- js/smoke/tests/deno/deno.json | 27 - 29 files changed, 3090 insertions(+), 329 deletions(-) create mode 100644 e2e/helpers/deno-test-helpers.ts create mode 100644 e2e/scenarios/deno-browser/__snapshots__/log-payloads.json create mode 100644 e2e/scenarios/deno-browser/__snapshots__/request-flow.json create mode 100644 e2e/scenarios/deno-browser/__snapshots__/span-events.json create mode 100644 e2e/scenarios/deno-browser/runner.case.ts create mode 100644 e2e/scenarios/deno-browser/scenario.test.ts create mode 100644 e2e/scenarios/deno-node/__snapshots__/log-payloads.json create mode 100644 e2e/scenarios/deno-node/__snapshots__/request-flow.json create mode 100644 e2e/scenarios/deno-node/__snapshots__/span-events.json create mode 100644 e2e/scenarios/deno-node/runner.case.ts create mode 100644 e2e/scenarios/deno-node/scenario.test.ts delete mode 100644 js/smoke/scenarios/deno-browser/.gitignore delete mode 100644 js/smoke/scenarios/deno-browser/Makefile delete mode 100644 js/smoke/scenarios/deno-browser/README.md delete mode 100644 js/smoke/scenarios/deno-browser/deno.json delete mode 100644 js/smoke/scenarios/deno-browser/mise.toml delete mode 100644 js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts delete mode 100644 js/smoke/scenarios/deno-node/.gitignore delete mode 100644 js/smoke/scenarios/deno-node/Makefile delete mode 100644 js/smoke/scenarios/deno-node/README.md delete mode 100644 js/smoke/scenarios/deno-node/deno.json delete mode 100644 js/smoke/scenarios/deno-node/mise.toml delete mode 100644 js/smoke/scenarios/deno-node/tests/shared-suite.test.ts delete mode 100644 js/smoke/tests/deno/deno.json diff --git a/.agents/skills/e2e-tests/SKILL.md b/.agents/skills/e2e-tests/SKILL.md index a0495ce80..89f6da879 100644 --- a/.agents/skills/e2e-tests/SKILL.md +++ b/.agents/skills/e2e-tests/SKILL.md @@ -14,6 +14,7 @@ pnpm run build # Build SDK (required if source changed) cd e2e && npx vitest run scenarios//scenario.test.ts # Run one scenario cd e2e && npx vitest run --reporter=verbose scenarios//scenario.test.ts # Verbose cd e2e && npx vitest run --update scenarios//scenario.test.ts # Update snapshots +cd e2e && npx vitest run -t "" # Isolate one test when file args over-match pnpm run test:e2e # Run all (from repo root) pnpm run test:e2e:hermetic # Run hermetic-only e2e tests pnpm run test:e2e:external # Run external-api-only e2e tests @@ -67,7 +68,7 @@ test( ); ``` -Key harness methods: `runScenarioDir()`, `runNodeScenarioDir()`, `testRunEvents()`, `events()`, `payloads()`, `requestsAfter(cursor)`, `testRunId`. +Key harness methods: `runScenarioDir()`, `runNodeScenarioDir()`, `runDenoScenarioDir()`, `testRunEvents()`, `events()`, `payloads()`, `requestsAfter(cursor)`, `testRunId`. For wrapper scenarios use `events()` (not `testRunEvents()`) and scope payloads via `payloadRowsForRootSpan()`. @@ -123,10 +124,18 @@ import { runMyImpl } from "./scenario.impl"; Test loops over versions with `for (const s of scenarios) { test(...) }`. See `wrap-ai-sdk-generation-traces` or `ai-sdk-otel-export`. -### Runner-wrapper (vitest/node:test) +### Runner-wrapper (vitest/node:test/deno) When the wrapper runs inside a nested test runner, `scenario.ts` spawns a second process via `runNodeSubprocess`. The nested runner file must NOT be named `*.test.ts`. Tag all data with `metadata.testRunId` and use `payloadRowsForTestRunId()`. See `wrap-vitest-suite-traces`. +Use: + +- `runNodeScenarioDir()` for plain Node nested runners +- `runDenoScenarioDir()` for Deno nested runners +- `runner.case.ts` for nested Deno entrypoints + +Deno scenarios can have intentionally different runtime contracts from Node. Assert the actual Deno/browser behavior rather than copying Node parent-child expectations blindly. See `e2e/scenarios/deno-browser/`. + ### OTEL export Set up `BraintrustExporter`/`BraintrustSpanProcessor` pointed at the mock server, register globally, then assert on `/otel/v1/traces` requests via `requestsAfter()` + `extractOtelSpans()`. See `ai-sdk-otel-export` or `otel-span-processor-export`. @@ -145,6 +154,8 @@ Set up `BraintrustExporter`/`BraintrustSpanProcessor` pointed at the mock server Scenarios run from `e2e/.bt-tmp/run-/scenarios//`. Node walks up to `e2e/node_modules/` for workspace deps (`braintrust`, `@braintrust/otel`, etc.). Scenario-local deps are in the scenario's own `node_modules/`. Helper imports (`../../helpers/...`) work because `prepareScenarioDir` copies `e2e/helpers/` into the temp dir. +Deno nested runners use `runDenoScenarioDir()`, which invokes `deno test --no-check` with the harness env vars and the prepared temp scenario path. + ## Debugging - **Subprocess error**: Read the `STDERR` section in the error message. diff --git a/e2e/README.md b/e2e/README.md index 25efd5b5d..1a79fac06 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -60,6 +60,7 @@ The main utilities you'll use in test files: - `resolveScenarioDir(import.meta.url)` - Resolves the folder that contains the current test. - `installScenarioDependencies({ scenarioDir })` - Installs optional scenario-local dependencies. - `runScenarioDir({ scenarioDir, entry?, timeoutMs? })` - Runs a TypeScript scenario with `tsx`. +- `runDenoScenarioDir({ scenarioDir, entry?, args?, timeoutMs? })` - Runs nested Deno scenarios with `deno test`. - `runNodeScenarioDir({ scenarioDir, entry?, nodeArgs?, timeoutMs? })` - Runs plain Node scenarios, used for `--import braintrust/hook.mjs`. - `testRunEvents()` - Returns parsed events tagged with the current test run id. - `events()`, `payloads()`, `requestCursor()`, `requestsAfter()` - Lower-level access for ingestion payloads and HTTP request flow assertions. @@ -108,6 +109,8 @@ Some wrappers execute inside a nested test runner rather than a single SDK call. - Tag every traced test/eval with `metadata.testRunId` so the outer assertions can isolate rows across multiple trace roots with `payloadRowsForTestRunId(...)`. - If a nested runner needs its own test discovery rules, keep that config local to the scenario folder so the shared e2e config stays unchanged. +The Deno scenarios follow the same pattern, except the harness invokes `deno test` via `runDenoScenarioDir(...)` and the nested runner entrypoint lives in `runner.case.ts`. + ### Environment variables `externalApi` scenarios require provider credentials in addition to the mock Braintrust server config supplied by the harness: diff --git a/e2e/helpers/deno-test-helpers.ts b/e2e/helpers/deno-test-helpers.ts new file mode 100644 index 000000000..f9a4e7093 --- /dev/null +++ b/e2e/helpers/deno-test-helpers.ts @@ -0,0 +1,192 @@ +type BraintrustModule = Record; + +function assert( + condition: unknown, + message: string, +): asserts condition is true { + if (!condition) { + throw new Error(message); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function getTestRunId(): string { + const testRunId = Deno.env.get("BRAINTRUST_E2E_RUN_ID"); + assert(testRunId, "BRAINTRUST_E2E_RUN_ID must be set"); + return testRunId; +} + +export function scopedName(base: string): string { + return `${base}-${getTestRunId() + .toLowerCase() + .replace(/[^a-z0-9-]/g, "-")}`; +} + +export function expectNamedExports( + module: BraintrustModule, + exportNames: string[], +): void { + for (const exportName of exportNames) { + assert(module[exportName], `Expected export "${exportName}" to exist`); + } +} + +export function expectBuildType( + module: BraintrustModule, + expectedBuildType: string, +): void { + const testingOnly = module._exportsForTestingOnly; + assert(isRecord(testingOnly), "_exportsForTestingOnly must exist"); + + const isomorph = testingOnly.isomorph; + assert(isRecord(isomorph), "_exportsForTestingOnly.isomorph must exist"); + assert( + isomorph.buildType === expectedBuildType, + `Expected build type "${expectedBuildType}" but got "${String(isomorph.buildType)}"`, + ); +} + +export function expectMustacheTemplate(module: BraintrustModule): void { + const Prompt = module.Prompt as + | (new (...args: unknown[]) => { + build: ( + args: Record, + options: { templateFormat: string }, + ) => { messages?: Array<{ content?: string }> }; + }) + | undefined; + + assert(Prompt, "Prompt export must exist"); + + const prompt = new Prompt( + { + name: "mustache-test", + slug: "mustache-test", + prompt_data: { + prompt: { + type: "chat", + messages: [{ role: "user", content: "Hello, {{name}}!" }], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + const result = prompt.build( + { name: "World" }, + { templateFormat: "mustache" }, + ); + assert( + result.messages?.[0]?.content === "Hello, World!", + "Mustache template rendering failed", + ); +} + +export function expectNunjucksTemplateUnavailable( + module: BraintrustModule, +): void { + const Prompt = module.Prompt as + | (new (...args: unknown[]) => { + build: ( + args: Record, + options: { templateFormat: string }, + ) => unknown; + }) + | undefined; + + assert(Prompt, "Prompt export must exist"); + + const prompt = new Prompt( + { + name: "nunjucks-test", + slug: "nunjucks-test", + prompt_data: { + prompt: { + type: "chat", + messages: [ + { + role: "user", + content: + "Items: {% for item in items %}{{ item.name }}{% if not loop.last %}, {% endif %}{% endfor %}", + }, + ], + }, + options: { model: "gpt-4" }, + }, + }, + {}, + false, + ); + + let errorMessage: string | undefined; + try { + prompt.build( + { + items: [{ name: "apple" }, { name: "banana" }, { name: "cherry" }], + }, + { templateFormat: "nunjucks" }, + ); + } catch (error) { + errorMessage = error instanceof Error ? error.message : String(error); + } + + assert( + errorMessage?.includes("requires @braintrust/template-nunjucks"), + `Expected missing nunjucks package error, got: ${errorMessage ?? "no error"}`, + ); +} + +export async function expectEvalWorks(module: BraintrustModule): Promise { + const Eval = module.Eval as + | (( + name: string, + definition: Record, + options: Record, + ) => Promise>) + | undefined; + + assert(Eval, "Eval export must exist"); + + const evalData = [ + { input: "Alice", expected: "Hi Alice" }, + { input: "Bob", expected: "Hi Bob" }, + { input: "Charlie", expected: "Hi Charlie" }, + ]; + + const result = await Eval( + "deno-local-eval", + { + data: evalData, + task: async (input: string) => `Hi ${input}`, + scores: [ + ({ expected, output }: { expected: string; output: string }) => ({ + name: "exact_match", + score: output === expected ? 1 : 0, + }), + ], + }, + { + noSendLogs: true, + returnResults: true, + }, + ); + + const summary = result.summary; + const results = result.results; + assert(Array.isArray(results), "Eval results must be an array"); + assert( + results.length === evalData.length, + "Eval returned the wrong row count", + ); + assert(isRecord(summary), "Eval summary must exist"); + assert(isRecord(summary.scores), "Eval summary scores must exist"); + + const exactMatch = summary.scores.exact_match; + assert(isRecord(exactMatch), "Eval exact_match summary must exist"); + assert(exactMatch.score === 1, "Eval exact_match summary must be 1"); +} diff --git a/e2e/helpers/scenario-harness.ts b/e2e/helpers/scenario-harness.ts index f825facb7..5c4fa62c1 100644 --- a/e2e/helpers/scenario-harness.ts +++ b/e2e/helpers/scenario-harness.ts @@ -29,6 +29,7 @@ interface ScenarioResult { } const tsxCliPath = createRequire(import.meta.url).resolve("tsx/cli"); +const DENO_COMMAND = process.platform === "win32" ? "deno.exe" : "deno"; const DEFAULT_SCENARIO_TIMEOUT_MS = 15_000; const HELPERS_DIR = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HELPERS_DIR, "../.."); @@ -77,13 +78,14 @@ function getTestServerEnv( } async function runProcess( + command: string, args: string[], cwd: string, env: Record, timeoutMs: number, ): Promise { return await new Promise((resolve, reject) => { - const child = spawn(process.execPath, args, { + const child = spawn(command, args, { cwd, env: { ...process.env, @@ -94,7 +96,9 @@ async function runProcess( const timeout = setTimeout(() => { child.kill("SIGTERM"); reject( - new Error(`Process ${args.join(" ")} timed out after ${timeoutMs}ms`), + new Error( + `Process ${command} ${args.join(" ")} timed out after ${timeoutMs}ms`, + ), ); }, timeoutMs); @@ -146,6 +150,7 @@ async function runScenarioDirOrThrow( ? [...(options.nodeArgs ?? []), scenarioPath] : [tsxCliPath, scenarioPath]; const result = await runProcess( + process.execPath, args, scenarioDir, env, @@ -192,6 +197,39 @@ export async function runNodeScenarioDir(options: { }); } +export async function runDenoScenarioDir(options: { + args?: string[]; + entry?: string; + env?: Record; + scenarioDir: string; + timeoutMs?: number; +}): Promise { + const entry = options.entry ?? "runner.case.ts"; + const result = await runProcess( + DENO_COMMAND, + [ + "test", + "--no-check", + "--allow-env", + "--allow-net", + "--allow-read", + ...(options.args ?? []), + resolveEntryPath(options.scenarioDir, entry), + ], + options.scenarioDir, + options.env ?? {}, + options.timeoutMs ?? DEFAULT_SCENARIO_TIMEOUT_MS, + ); + + if (result.exitCode !== 0) { + throw new Error( + `Scenario ${path.join(options.scenarioDir, entry)} failed with exit code ${result.exitCode}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`, + ); + } + + return result; +} + interface ScenarioHarness { events: (predicate?: EventPredicate) => CapturedLogEvent[]; payloads: (predicate?: PayloadPredicate) => CapturedLogPayload[]; @@ -200,6 +238,13 @@ interface ScenarioHarness { after: number, predicate?: RequestPredicate, ) => CapturedRequest[]; + runDenoScenarioDir: (options: { + args?: string[]; + entry?: string; + env?: Record; + scenarioDir: string; + timeoutMs?: number; + }) => Promise; runNodeScenarioDir: (options: { entry?: string; env?: Record; @@ -231,6 +276,14 @@ export async function withScenarioHarness( requestCursor: () => server.requests.length, requestsAfter: (after, predicate) => filterItems(server.requests.slice(after), predicate), + runDenoScenarioDir: (options) => + runDenoScenarioDir({ + ...options, + env: { + ...testEnv, + ...(options.env ?? {}), + }, + }), runNodeScenarioDir: (options) => runNodeScenarioDir({ ...options, diff --git a/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json b/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json new file mode 100644 index 000000000..002384c8a --- /dev/null +++ b/e2e/scenarios/deno-browser/__snapshots__/log-payloads.json @@ -0,0 +1,242 @@ +[ + { + "_is_merge": false, + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno browser basic span", + "type": "task" + }, + "span_id": "" + }, + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "", + "transport": "http" + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metrics": { + "end": 0 + }, + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno browser json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno browser parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno browser child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno browser nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno browser nested child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno browser nested grandchild span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno browser current span", + "type": "task" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/deno-browser/__snapshots__/request-flow.json b/e2e/scenarios/deno-browser/__snapshots__/request-flow.json new file mode 100644 index 000000000..28a1c00fd --- /dev/null +++ b/e2e/scenarios/deno-browser/__snapshots__/request-flow.json @@ -0,0 +1,520 @@ +[ + { + "headers": null, + "jsonBody": null, + "method": "POST", + "path": "/api/apikey/login", + "query": null, + "rawBody": null + }, + { + "headers": null, + "jsonBody": { + "org_id": "mock-org-id", + "project_name": "e2e-deno-browser-e2e-" + }, + "method": "POST", + "path": "/api/project/register", + "query": null, + "rawBody": { + "org_id": "mock-org-id", + "project_name": "e2e-deno-browser-e2e-" + } + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno browser basic span", + "type": "task" + }, + "span_id": "" + } + ] + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno browser basic span", + "type": "task" + }, + "span_id": "" + } + ] + } + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "", + "transport": "http" + }, + "metrics": { + "end": 0 + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno browser json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno browser parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno browser child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno browser nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno browser nested child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno browser nested grandchild span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno browser current span", + "type": "task" + }, + "span_id": "" + } + ] + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "", + "transport": "http" + }, + "metrics": { + "end": 0 + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno browser json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno browser parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno browser child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno browser nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno browser nested child span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno browser nested grandchild span", + "type": "task" + }, + "span_id": "" + }, + { + "context": {}, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno browser current span", + "type": "task" + }, + "span_id": "" + } + ] + } + } +] diff --git a/e2e/scenarios/deno-browser/__snapshots__/span-events.json b/e2e/scenarios/deno-browser/__snapshots__/span-events.json new file mode 100644 index 000000000..41fcf80f0 --- /dev/null +++ b/e2e/scenarios/deno-browser/__snapshots__/span-events.json @@ -0,0 +1,177 @@ +[ + { + "error": null, + "input": "What is the capital of France?", + "metadata": { + "case": "basic-span", + "scenario": "deno-browser", + "testRunId": "", + "transport": "http" + }, + "name": "deno browser basic span", + "output": "Paris", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno browser basic span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "metadata": { + "case": "json-attachment", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser json attachment span", + "output": { + "attachment": true + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno browser json attachment span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "phase": "parent", + "testRunId": "" + }, + "metadata": { + "case": "parent-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser parent span", + "output": { + "ok": true, + "phase": "parent" + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno browser parent span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "step": "child", + "testRunId": "" + }, + "metadata": { + "case": "child-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser child span", + "output": { + "ok": true, + "phase": "child" + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno browser child span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-parent", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser nested parent span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno browser nested parent span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-child", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser nested child span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno browser nested child span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser nested grandchild span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno browser nested grandchild span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": null, + "metadata": { + "case": "current-span", + "scenario": "deno-browser", + "testRunId": "" + }, + "name": "deno browser current span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno browser current span", + "type": "task" + }, + "span_id": "", + "span_parents": null + } +] diff --git a/e2e/scenarios/deno-browser/runner.case.ts b/e2e/scenarios/deno-browser/runner.case.ts new file mode 100644 index 000000000..88197b73b --- /dev/null +++ b/e2e/scenarios/deno-browser/runner.case.ts @@ -0,0 +1,208 @@ +import * as braintrust from "braintrust/browser"; +import { + expectBuildType, + expectEvalWorks, + expectMustacheTemplate, + expectNamedExports, + expectNunjucksTemplateUnavailable, + getTestRunId, + scopedName, +} from "../../helpers/deno-test-helpers.ts"; + +const scenario = "deno-browser"; +const testRunId = getTestRunId(); + +function createLogger() { + return braintrust.initLogger({ + projectName: scopedName("e2e-deno-browser"), + }); +} + +function metadata(caseName: string) { + return { case: caseName, scenario, testRunId }; +} + +Deno.test("deno-browser exposes the expected browser runtime surface", () => { + expectNamedExports(braintrust, [ + "initLogger", + "currentSpan", + "startSpan", + "log", + "flush", + "initDataset", + "initExperiment", + "loadPrompt", + "Prompt", + "Eval", + "traced", + "wrapOpenAI", + "JSONAttachment", + "Attachment", + "invoke", + "initFunction", + "Project", + "PromptBuilder", + "IDGenerator", + "BraintrustState", + "login", + "_exportsForTestingOnly", + ]); + expectBuildType(braintrust, "browser"); + expectMustacheTemplate(braintrust); + expectNunjucksTemplateUnavailable(braintrust); +}); + +Deno.test( + "deno-browser can run Eval locally without sending logs", + async () => { + await expectEvalWorks(braintrust); + }, +); + +Deno.test("deno-browser emits trace payloads over HTTP", async () => { + const logger = createLogger(); + + const basicSpan = logger.startSpan({ + name: "deno browser basic span", + event: { + metadata: metadata("basic-span"), + }, + }); + basicSpan.log({ + expected: "Paris", + input: "What is the capital of France?", + metadata: { + ...metadata("basic-span"), + transport: "http", + }, + output: "Paris", + }); + basicSpan.end(); + + const attachmentSpan = logger.startSpan({ + name: "deno browser json attachment span", + event: { + metadata: metadata("json-attachment"), + }, + }); + attachmentSpan.log({ + input: { + transcript: new braintrust.JSONAttachment( + { + foo: "bar", + nested: { + array: [1, 2, 3], + ok: true, + }, + }, + { + filename: "conversation_transcript.json", + pretty: true, + }, + ), + type: "chat_completion", + }, + metadata: metadata("json-attachment"), + output: { + attachment: true, + }, + }); + attachmentSpan.end(); + + await braintrust.traced( + async (parentSpan) => { + const childSpan = braintrust.startSpan({ + name: "deno browser child span", + event: { + input: { + step: "child", + testRunId, + }, + metadata: metadata("child-span"), + }, + }); + + childSpan.log({ + output: { + phase: "child", + ok: true, + }, + }); + childSpan.end(); + + parentSpan.log({ + output: { + phase: "parent", + ok: true, + }, + }); + }, + { + name: "deno browser parent span", + event: { + input: { + phase: "parent", + testRunId, + }, + metadata: metadata("parent-span"), + }, + }, + ); + + await braintrust.traced( + async () => { + await braintrust.traced( + async () => { + await braintrust.traced( + async () => { + braintrust.currentSpan().log({ + metadata: metadata("nested-grandchild"), + output: { + depth: 3, + }, + }); + }, + { + name: "deno browser nested grandchild span", + event: { + metadata: metadata("nested-grandchild"), + }, + }, + ); + }, + { + name: "deno browser nested child span", + event: { + metadata: metadata("nested-child"), + }, + }, + ); + }, + { + name: "deno browser nested parent span", + event: { + metadata: metadata("nested-parent"), + }, + }, + ); + + await braintrust.traced( + async () => { + const activeSpan = braintrust.currentSpan(); + activeSpan.log({ + metadata: metadata("current-span"), + output: { + observedSpanId: activeSpan.spanId, + }, + }); + }, + { + name: "deno browser current span", + event: { + metadata: metadata("current-span"), + }, + }, + ); + + await logger.flush(); +}); diff --git a/e2e/scenarios/deno-browser/scenario.test.ts b/e2e/scenarios/deno-browser/scenario.test.ts new file mode 100644 index 000000000..cbf2d16d5 --- /dev/null +++ b/e2e/scenarios/deno-browser/scenario.test.ts @@ -0,0 +1,191 @@ +import { expect, test } from "vitest"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import type { Json } from "../../helpers/normalize"; +import { + prepareScenarioDir, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { E2E_TAGS } from "../../helpers/tags"; +import { findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; +import { payloadRowsForTestRunId } from "../../helpers/wrapper-contract"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); +const TIMEOUT_MS = 90_000; + +function findEventByCase(events: CapturedLogEvent[], testCase: string) { + return events.find((event) => { + const metadata = event.row.metadata as Record | undefined; + return metadata?.case === testCase; + }); +} + +test( + "deno-browser captures real HTTP traces from the browser build in Deno", + { + tags: [E2E_TAGS.hermetic], + timeout: TIMEOUT_MS, + }, + async () => { + await withScenarioHarness( + async ({ + payloads, + requestCursor, + requestsAfter, + runDenoScenarioDir, + testRunEvents, + testRunId, + }) => { + const cursor = requestCursor(); + + await runDenoScenarioDir({ + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + + const capturedEvents = testRunEvents(); + const basicSpan = findLatestSpan( + capturedEvents, + "deno browser basic span", + ); + const jsonAttachment = findEventByCase( + capturedEvents, + "json-attachment", + ); + const parentSpan = findLatestSpan( + capturedEvents, + "deno browser parent span", + ); + const childSpan = findLatestSpan( + capturedEvents, + "deno browser child span", + ); + const nestedParent = findLatestSpan( + capturedEvents, + "deno browser nested parent span", + ); + const nestedChild = findLatestSpan( + capturedEvents, + "deno browser nested child span", + ); + const nestedGrandchild = findLatestSpan( + capturedEvents, + "deno browser nested grandchild span", + ); + const currentSpan = findLatestSpan( + capturedEvents, + "deno browser current span", + ); + + for (const span of [ + basicSpan, + jsonAttachment, + parentSpan, + childSpan, + nestedParent, + nestedChild, + nestedGrandchild, + currentSpan, + ]) { + expect(span).toBeDefined(); + } + + expect(basicSpan?.input).toBe("What is the capital of France?"); + expect(basicSpan?.output).toBe("Paris"); + expect(basicSpan?.expected).toBe("Paris"); + expect(basicSpan?.row.metadata).toMatchObject({ + case: "basic-span", + scenario: "deno-browser", + testRunId, + transport: "http", + }); + + expect(jsonAttachment?.input).toMatchObject({ + type: "chat_completion", + }); + expect(jsonAttachment?.row.metadata).toMatchObject({ + case: "json-attachment", + scenario: "deno-browser", + testRunId, + }); + expect(jsonAttachment?.output).toMatchObject({ + attachment: true, + }); + + expect(parentSpan?.output).toMatchObject({ + phase: "parent", + ok: true, + }); + expect(childSpan?.output).toMatchObject({ + phase: "child", + ok: true, + }); + expect(childSpan?.span.parentIds).toEqual([]); + expect(nestedChild?.span.parentIds).toEqual([]); + expect(nestedGrandchild?.span.parentIds).toEqual([]); + expect(nestedGrandchild?.output).toBeUndefined(); + expect(currentSpan?.output).toBeUndefined(); + + const requests = requestsAfter( + cursor, + (request) => + request.path === "/api/apikey/login" || + request.path === "/api/project/register" || + request.path === "/logs3", + ); + + expect(requests.map((request) => request.path)).toEqual( + expect.arrayContaining([ + "/api/apikey/login", + "/api/project/register", + "/logs3", + ]), + ); + + await expect( + formatJsonFileSnapshot( + [ + basicSpan, + jsonAttachment, + parentSpan, + childSpan, + nestedParent, + nestedChild, + nestedGrandchild, + currentSpan, + ].map((event) => summarizeEvent(event!)) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "span-events.json"), + ); + + await expect( + formatJsonFileSnapshot( + payloadRowsForTestRunId(payloads(), testRunId) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + ); + + await expect( + formatJsonFileSnapshot( + requests.map((request) => + summarizeRequest(request, { + normalizeJsonRawBody: true, + }), + ) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "request-flow.json"), + ); + }, + ); + }, +); diff --git a/e2e/scenarios/deno-node/__snapshots__/log-payloads.json b/e2e/scenarios/deno-node/__snapshots__/log-payloads.json new file mode 100644 index 000000000..de2b665a5 --- /dev/null +++ b/e2e/scenarios/deno-node/__snapshots__/log-payloads.json @@ -0,0 +1,286 @@ +[ + { + "_is_merge": false, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno node basic span", + "type": "task" + }, + "span_id": "" + }, + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "", + "transport": "http" + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "_is_merge": true, + "id": "", + "log_id": "g", + "metrics": { + "end": 0 + }, + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno node json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno node parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno node child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno node nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno node nested child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "depth": 3 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno node nested grandchild span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "observedSpanId": "" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno node current span", + "type": "task" + }, + "span_id": "" + } +] diff --git a/e2e/scenarios/deno-node/__snapshots__/request-flow.json b/e2e/scenarios/deno-node/__snapshots__/request-flow.json new file mode 100644 index 000000000..afa4eee3c --- /dev/null +++ b/e2e/scenarios/deno-node/__snapshots__/request-flow.json @@ -0,0 +1,608 @@ +[ + { + "headers": null, + "jsonBody": null, + "method": "POST", + "path": "/api/apikey/login", + "query": null, + "rawBody": null + }, + { + "headers": null, + "jsonBody": { + "org_id": "mock-org-id", + "project_name": "e2e-deno-node-e2e-" + }, + "method": "POST", + "path": "/api/project/register", + "query": null, + "rawBody": { + "org_id": "mock-org-id", + "project_name": "e2e-deno-node-e2e-" + } + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno node basic span", + "type": "task" + }, + "span_id": "" + } + ] + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": false, + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno node basic span", + "type": "task" + }, + "span_id": "" + } + ] + } + }, + { + "headers": null, + "jsonBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "", + "transport": "http" + }, + "metrics": { + "end": 0 + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno node json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno node parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno node child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno node nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno node nested child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "depth": 3 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno node nested grandchild span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "observedSpanId": "" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno node current span", + "type": "task" + }, + "span_id": "" + } + ] + }, + "method": "POST", + "path": "/logs3", + "query": null, + "rawBody": { + "api_version": 2, + "rows": [ + { + "_is_merge": true, + "expected": "Paris", + "id": "", + "input": "What is the capital of France?", + "log_id": "g", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "", + "transport": "http" + }, + "metrics": { + "end": 0 + }, + "output": "Paris", + "project_id": "", + "root_span_id": "", + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "log_id": "g", + "metadata": { + "case": "json-attachment", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "attachment": true + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno node json attachment span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "phase": "parent", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "parent-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "parent" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno node parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "input": { + "step": "child", + "testRunId": "" + }, + "log_id": "g", + "metadata": { + "case": "child-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "ok": true, + "phase": "child" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno node child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-parent", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno node nested parent span", + "type": "task" + }, + "span_id": "" + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-child", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno node nested child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "/e2e/scenarios/deno-node/runner.case.ts", + "caller_functionname": "braintrust.traced.name", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "depth": 3 + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno node nested grandchild span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "context": { + "caller_filename": "ext:cli/40_test.js", + "caller_functionname": "async innerWrapped", + "caller_lineno": 0 + }, + "created": "", + "id": "", + "log_id": "g", + "metadata": { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + }, + "metrics": { + "end": 0, + "start": 0 + }, + "output": { + "observedSpanId": "" + }, + "project_id": "", + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno node current span", + "type": "task" + }, + "span_id": "" + } + ] + } + } +] diff --git a/e2e/scenarios/deno-node/__snapshots__/span-events.json b/e2e/scenarios/deno-node/__snapshots__/span-events.json new file mode 100644 index 000000000..7c1b0e158 --- /dev/null +++ b/e2e/scenarios/deno-node/__snapshots__/span-events.json @@ -0,0 +1,184 @@ +[ + { + "error": null, + "input": "What is the capital of France?", + "metadata": { + "case": "basic-span", + "scenario": "deno-node", + "testRunId": "", + "transport": "http" + }, + "name": "deno node basic span", + "output": "Paris", + "root_span_id": "", + "span_attributes": { + "exec_counter": 0, + "name": "deno node basic span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "transcript": { + "content_type": "application/json", + "filename": "conversation_transcript.json", + "key": "", + "type": "braintrust_attachment" + }, + "type": "chat_completion" + }, + "metadata": { + "case": "json-attachment", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node json attachment span", + "output": { + "attachment": true + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 1, + "name": "deno node json attachment span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "phase": "parent", + "testRunId": "" + }, + "metadata": { + "case": "parent-span", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node parent span", + "output": { + "ok": true, + "phase": "parent" + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 2, + "name": "deno node parent span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": { + "step": "child", + "testRunId": "" + }, + "metadata": { + "case": "child-span", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node child span", + "output": { + "ok": true, + "phase": "child" + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 3, + "name": "deno node child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-parent", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node nested parent span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 4, + "name": "deno node nested parent span", + "type": "task" + }, + "span_id": "", + "span_parents": null + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-child", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node nested child span", + "output": null, + "root_span_id": "", + "span_attributes": { + "exec_counter": 5, + "name": "deno node nested child span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "error": null, + "input": null, + "metadata": { + "case": "nested-grandchild", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node nested grandchild span", + "output": { + "depth": 3 + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 6, + "name": "deno node nested grandchild span" + }, + "span_id": "", + "span_parents": [ + "" + ] + }, + { + "error": null, + "input": null, + "metadata": { + "case": "current-span", + "scenario": "deno-node", + "testRunId": "" + }, + "name": "deno node current span", + "output": { + "observedSpanId": "" + }, + "root_span_id": "", + "span_attributes": { + "exec_counter": 7, + "name": "deno node current span", + "type": "task" + }, + "span_id": "", + "span_parents": null + } +] diff --git a/e2e/scenarios/deno-node/runner.case.ts b/e2e/scenarios/deno-node/runner.case.ts new file mode 100644 index 000000000..b8f88b2db --- /dev/null +++ b/e2e/scenarios/deno-node/runner.case.ts @@ -0,0 +1,205 @@ +import * as braintrust from "braintrust"; +import { + expectBuildType, + expectEvalWorks, + expectMustacheTemplate, + expectNamedExports, + expectNunjucksTemplateUnavailable, + getTestRunId, + scopedName, +} from "../../helpers/deno-test-helpers.ts"; + +const scenario = "deno-node"; +const testRunId = getTestRunId(); + +function createLogger() { + return braintrust.initLogger({ + projectName: scopedName("e2e-deno-node"), + }); +} + +function metadata(caseName: string) { + return { case: caseName, scenario, testRunId }; +} + +Deno.test("deno-node exposes the expected node runtime surface", () => { + expectNamedExports(braintrust, [ + "initLogger", + "currentSpan", + "startSpan", + "log", + "flush", + "initDataset", + "initExperiment", + "loadPrompt", + "Prompt", + "Eval", + "traced", + "wrapOpenAI", + "JSONAttachment", + "Attachment", + "invoke", + "initFunction", + "Project", + "PromptBuilder", + "IDGenerator", + "BraintrustState", + "login", + "_exportsForTestingOnly", + ]); + expectBuildType(braintrust, "node"); + expectMustacheTemplate(braintrust); + expectNunjucksTemplateUnavailable(braintrust); +}); + +Deno.test("deno-node can run Eval locally without sending logs", async () => { + await expectEvalWorks(braintrust); +}); + +Deno.test("deno-node emits trace payloads over HTTP", async () => { + const logger = createLogger(); + + const basicSpan = logger.startSpan({ + name: "deno node basic span", + event: { + metadata: metadata("basic-span"), + }, + }); + basicSpan.log({ + expected: "Paris", + input: "What is the capital of France?", + metadata: { + ...metadata("basic-span"), + transport: "http", + }, + output: "Paris", + }); + basicSpan.end(); + + const attachmentSpan = logger.startSpan({ + name: "deno node json attachment span", + event: { + metadata: metadata("json-attachment"), + }, + }); + attachmentSpan.log({ + input: { + transcript: new braintrust.JSONAttachment( + { + foo: "bar", + nested: { + array: [1, 2, 3], + ok: true, + }, + }, + { + filename: "conversation_transcript.json", + pretty: true, + }, + ), + type: "chat_completion", + }, + metadata: metadata("json-attachment"), + output: { + attachment: true, + }, + }); + attachmentSpan.end(); + + await braintrust.traced( + async (parentSpan) => { + const childSpan = braintrust.startSpan({ + name: "deno node child span", + event: { + input: { + step: "child", + testRunId, + }, + metadata: metadata("child-span"), + }, + }); + + childSpan.log({ + output: { + phase: "child", + ok: true, + }, + }); + childSpan.end(); + + parentSpan.log({ + output: { + phase: "parent", + ok: true, + }, + }); + }, + { + name: "deno node parent span", + event: { + input: { + phase: "parent", + testRunId, + }, + metadata: metadata("parent-span"), + }, + }, + ); + + await braintrust.traced( + async () => { + await braintrust.traced( + async () => { + await braintrust.traced( + async () => { + braintrust.currentSpan().log({ + metadata: metadata("nested-grandchild"), + output: { + depth: 3, + }, + }); + }, + { + name: "deno node nested grandchild span", + event: { + metadata: metadata("nested-grandchild"), + }, + }, + ); + }, + { + name: "deno node nested child span", + event: { + metadata: metadata("nested-child"), + }, + }, + ); + }, + { + name: "deno node nested parent span", + event: { + metadata: metadata("nested-parent"), + }, + }, + ); + + await braintrust.traced( + async () => { + const activeSpan = braintrust.currentSpan(); + activeSpan.log({ + metadata: metadata("current-span"), + output: { + observedSpanId: activeSpan.spanId, + }, + }); + }, + { + name: "deno node current span", + event: { + metadata: metadata("current-span"), + }, + }, + ); + + await logger.flush(); +}); diff --git a/e2e/scenarios/deno-node/scenario.test.ts b/e2e/scenarios/deno-node/scenario.test.ts new file mode 100644 index 000000000..e4088101d --- /dev/null +++ b/e2e/scenarios/deno-node/scenario.test.ts @@ -0,0 +1,201 @@ +import { expect, test } from "vitest"; +import { + formatJsonFileSnapshot, + resolveFileSnapshotPath, +} from "../../helpers/file-snapshot"; +import type { CapturedLogEvent } from "../../helpers/mock-braintrust-server"; +import type { Json } from "../../helpers/normalize"; +import { + prepareScenarioDir, + resolveScenarioDir, + withScenarioHarness, +} from "../../helpers/scenario-harness"; +import { E2E_TAGS } from "../../helpers/tags"; +import { findLatestSpan } from "../../helpers/trace-selectors"; +import { summarizeEvent, summarizeRequest } from "../../helpers/trace-summary"; +import { payloadRowsForTestRunId } from "../../helpers/wrapper-contract"; + +const scenarioDir = await prepareScenarioDir({ + scenarioDir: resolveScenarioDir(import.meta.url), +}); +const TIMEOUT_MS = 90_000; + +function findEventByCase(events: CapturedLogEvent[], testCase: string) { + return events.find((event) => { + const metadata = event.row.metadata as Record | undefined; + return metadata?.case === testCase; + }); +} + +test( + "deno-node captures real HTTP traces from a nested Deno runner", + { + tags: [E2E_TAGS.hermetic], + timeout: TIMEOUT_MS, + }, + async () => { + await withScenarioHarness( + async ({ + payloads, + requestCursor, + requestsAfter, + runDenoScenarioDir, + testRunEvents, + testRunId, + }) => { + const cursor = requestCursor(); + + await runDenoScenarioDir({ + scenarioDir, + timeoutMs: TIMEOUT_MS, + }); + + const capturedEvents = testRunEvents(); + const basicSpan = findLatestSpan( + capturedEvents, + "deno node basic span", + ); + const jsonAttachment = findEventByCase( + capturedEvents, + "json-attachment", + ); + const parentSpan = findLatestSpan( + capturedEvents, + "deno node parent span", + ); + const childSpan = findLatestSpan( + capturedEvents, + "deno node child span", + ); + const nestedParent = findLatestSpan( + capturedEvents, + "deno node nested parent span", + ); + const nestedChild = findLatestSpan( + capturedEvents, + "deno node nested child span", + ); + const nestedGrandchild = findLatestSpan( + capturedEvents, + "deno node nested grandchild span", + ); + const currentSpan = findLatestSpan( + capturedEvents, + "deno node current span", + ); + + for (const span of [ + basicSpan, + jsonAttachment, + parentSpan, + childSpan, + nestedParent, + nestedChild, + nestedGrandchild, + currentSpan, + ]) { + expect(span).toBeDefined(); + } + + expect(basicSpan?.input).toBe("What is the capital of France?"); + expect(basicSpan?.output).toBe("Paris"); + expect(basicSpan?.expected).toBe("Paris"); + expect(basicSpan?.row.metadata).toMatchObject({ + case: "basic-span", + scenario: "deno-node", + testRunId, + transport: "http", + }); + + expect(jsonAttachment?.input).toMatchObject({ + type: "chat_completion", + }); + expect(jsonAttachment?.row.metadata).toMatchObject({ + case: "json-attachment", + scenario: "deno-node", + testRunId, + }); + expect(jsonAttachment?.output).toMatchObject({ + attachment: true, + }); + + expect(parentSpan?.output).toMatchObject({ + phase: "parent", + ok: true, + }); + expect(childSpan?.span.parentIds).toEqual([parentSpan?.span.id ?? ""]); + expect(childSpan?.output).toMatchObject({ + phase: "child", + ok: true, + }); + + expect(nestedChild?.span.parentIds).toEqual([ + nestedParent?.span.id ?? "", + ]); + expect(nestedGrandchild?.span.parentIds).toEqual([ + nestedChild?.span.id ?? "", + ]); + expect(nestedGrandchild?.output).toMatchObject({ + depth: 3, + }); + + expect(currentSpan?.output).toMatchObject({ + observedSpanId: currentSpan?.span.id, + }); + + const requests = requestsAfter( + cursor, + (request) => + request.path === "/api/apikey/login" || + request.path === "/api/project/register" || + request.path === "/logs3", + ); + + expect(requests.map((request) => request.path)).toEqual( + expect.arrayContaining([ + "/api/apikey/login", + "/api/project/register", + "/logs3", + ]), + ); + + await expect( + formatJsonFileSnapshot( + [ + basicSpan, + jsonAttachment, + parentSpan, + childSpan, + nestedParent, + nestedChild, + nestedGrandchild, + currentSpan, + ].map((event) => summarizeEvent(event!)) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "span-events.json"), + ); + + await expect( + formatJsonFileSnapshot( + payloadRowsForTestRunId(payloads(), testRunId) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "log-payloads.json"), + ); + + await expect( + formatJsonFileSnapshot( + requests.map((request) => + summarizeRequest(request, { + normalizeJsonRawBody: true, + }), + ) as Json, + ), + ).toMatchFileSnapshot( + resolveFileSnapshotPath(import.meta.url, "request-flow.json"), + ); + }, + ); + }, +); diff --git a/js/smoke/README.md b/js/smoke/README.md index 7bdad6a18..c9091b4f2 100644 --- a/js/smoke/README.md +++ b/js/smoke/README.md @@ -2,6 +2,8 @@ Smoke test infrastructure verifying SDK installation across different runtimes and integrations. +Deno coverage now lives in `e2e/scenarios/deno-node/` and `e2e/scenarios/deno-browser/`. + ## Quick Reference ```bash @@ -10,12 +12,11 @@ make test-smoke # Run all scenarios # From sdk/js/smoke/: make test # Run all scenarios (local + integration) -make test deno-node # Run specific local scenario make test templates-nunjucks/jest # Run integration scenario make list # List all discovered scenarios # From a specific scenario: -cd scenarios/deno-node +cd scenarios/ make test # Auto-creates tarball if needed ``` @@ -79,8 +80,6 @@ if (failed.length > 0) { **Node.js + npm:** Use tarball paths in package.json. See `scenarios/otel-v1/`. -**Deno:** Use workspace links via `deno.json`. See `scenarios/deno-node/`. - **Multi-test:** Each test uses `displayTestResults()`. Makefile runs all tests (don't fail early), then exits with failure if any failed. See `scenarios/otel-v1/` for example. ### Example Test File @@ -330,7 +329,6 @@ smoke/ │ │ ├── mise.toml # Environment definition │ │ ├── package.json # Dependencies only (no scripts) │ │ └── README.md -│ └── deno-node/ ├── Makefile # Top-level orchestration (auto-discovery) └── README.md # This file ``` @@ -359,7 +357,6 @@ See `shared/README.md` for complete test suite documentation, individual test fu ## Reference Scenarios - **Node.js + OTEL:** `scenarios/otel-v1/` -- **Deno:** `scenarios/deno-node/`, `scenarios/deno-browser/` - **Cloudflare Workers:** `scenarios/cloudflare-worker-*/` - **Multi-test:** `scenarios/cloudflare-vite-hono/` - **Next.js:** `scenarios/nextjs-instrumentation/` diff --git a/js/smoke/scenarios/deno-browser/.gitignore b/js/smoke/scenarios/deno-browser/.gitignore deleted file mode 100644 index 0203990b0..000000000 --- a/js/smoke/scenarios/deno-browser/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules/ -.DS_Store -package-lock.json -deno.lock diff --git a/js/smoke/scenarios/deno-browser/Makefile b/js/smoke/scenarios/deno-browser/Makefile deleted file mode 100644 index 8c95c70db..000000000 --- a/js/smoke/scenarios/deno-browser/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -.PHONY: setup test - -setup: - @echo "==> Setting up deno-browser scenario" - mise install - @# Deno uses workspace links - need SDK dist/ folder to exist - if [ -n "$(BRAINTRUST_TAR)" ]; then \ - echo "==> Extracting SDK tarball for workspace link"; \ - TARBALL_PATH="$$(cd ../.. && pwd)/$(BRAINTRUST_TAR)"; \ - echo "==> Resolved tarball path: $$TARBALL_PATH"; \ - cd ../../.. && tar -xzf "$$TARBALL_PATH" --strip-components=1 package/dist; \ - else \ - echo "==> Building SDK for workspace link"; \ - cd ../../.. && pnpm exec turbo build --filter=braintrust; \ - fi; \ - -test: setup - @# Run only tests under the local tests/ directory - mise exec -- deno test --sloppy-imports --allow-all tests/*.test.ts diff --git a/js/smoke/scenarios/deno-browser/README.md b/js/smoke/scenarios/deno-browser/README.md deleted file mode 100644 index 790322183..000000000 --- a/js/smoke/scenarios/deno-browser/README.md +++ /dev/null @@ -1,35 +0,0 @@ -# Deno Browser Build Test - -Tests the browser-specific build (`braintrust/browser` export) in Deno runtime. - -## Design Decisions - -### Local Package Linking - -Uses Deno's `links` feature in `deno.json` to test against local SDK builds: - -- `"links": ["../../..", "../../shared"]` ensures we use the workspace versions -- No need to publish or pack - Deno reads from `../../../dist/` directly - -### Sloppy Imports - -Requires `--sloppy-imports` flag because the shared test package uses extensionless imports: - -- Shared package: `import { foo } from "./helpers/types"` (no `.ts`) -- This is standard for TypeScript/Node.js but requires `--sloppy-imports` in Deno -- Alternative would be adding `.ts` extensions, but that may break Node.js tooling - -### npm Compatibility - -Uses `nodeModulesDir: "auto"` to enable Deno's npm package resolution: - -- Allows `npm:braintrust@^2.0.2/browser` imports -- Combined with `links`, resolves to local workspace packages - -### Browser vs Node Build - -Uses `braintrust/browser` export instead of main `braintrust`: - -- Browser-optimized bundle (no nunjucks, simple-git, etc.) -- Only mustache templates supported -- Platform-specific `.browser.ts` implementations diff --git a/js/smoke/scenarios/deno-browser/deno.json b/js/smoke/scenarios/deno-browser/deno.json deleted file mode 100644 index 4e38fc8f6..000000000 --- a/js/smoke/scenarios/deno-browser/deno.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "imports": { - "@std/assert": "jsr:@std/assert@^1.0.14", - "@braintrust/smoke-test-shared": "jsr:@braintrust/smoke-test-shared", - "braintrust": "npm:braintrust" - }, - "nodeModulesDir": "auto", - "links": ["../../..", "../../shared"] -} diff --git a/js/smoke/scenarios/deno-browser/mise.toml b/js/smoke/scenarios/deno-browser/mise.toml deleted file mode 100644 index dd563ce6f..000000000 --- a/js/smoke/scenarios/deno-browser/mise.toml +++ /dev/null @@ -1,3 +0,0 @@ -[tools] -deno = "latest" -pnpm = "10.26.2" diff --git a/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts b/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts deleted file mode 100644 index 56d2cb89e..000000000 --- a/js/smoke/scenarios/deno-browser/tests/shared-suite.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -// @ts-nocheck -/** - * Deno smoke test using shared test suites (browser build) - */ - -import { assertEquals } from "@std/assert"; -import { - runTests, - expectFailure, - testBasicSpanLogging, - testMultipleSpans, - testDirectLogging, - testJSONAttachment, - testAsyncLocalStorageTraced, - testNestedTraced, - testCurrentSpan, - testCoreLoggingExports, - testDatasetExports, - testPromptExports, - testExperimentExports, - testEvalExports, - testTracingExports, - testClientWrapperExports, - testUtilityExports, - testFunctionExports, - testFramework2Exports, - testIDGeneratorExports, - testTestingExports, - testStateManagementExports, - testBuildResolution, - testMustacheTemplate, - testNunjucksTemplate, - testEvalSmoke, -} from "@braintrust/smoke-test-shared"; -import * as braintrust from "braintrust"; - -Deno.test("Run shared test suites (browser build)", async () => { - const { failed } = await runTests({ - name: "deno-browser", - braintrust, - tests: [ - testCoreLoggingExports, - testDatasetExports, - testPromptExports, - testExperimentExports, - testEvalExports, - testTracingExports, - testClientWrapperExports, - testUtilityExports, - testFunctionExports, - testFramework2Exports, - testIDGeneratorExports, - testTestingExports, - testStateManagementExports, - testBuildResolution("node"), // not pulling in browser package - testBasicSpanLogging, - testMultipleSpans, - testDirectLogging, - testJSONAttachment, - expectFailure( - testAsyncLocalStorageTraced, - (e: { message: string }) => e.message.includes("span_parents"), - "No AsyncLocalStorage available", - ), - expectFailure( - testNestedTraced, - (e: { message: string }) => e.message.includes("span_parents"), - "No AsyncLocalStorage available", - ), - testCurrentSpan, - testEvalSmoke, - testMustacheTemplate, - expectFailure( - testNunjucksTemplate, - (e: { message: string }) => - e.message.includes("requires @braintrust/template-nunjucks"), - "Nunjucks requires separate package", - ), - ], - }); - - assertEquals(failed.length, 0, "All tests should pass"); -}); diff --git a/js/smoke/scenarios/deno-node/.gitignore b/js/smoke/scenarios/deno-node/.gitignore deleted file mode 100644 index 0203990b0..000000000 --- a/js/smoke/scenarios/deno-node/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules/ -.DS_Store -package-lock.json -deno.lock diff --git a/js/smoke/scenarios/deno-node/Makefile b/js/smoke/scenarios/deno-node/Makefile deleted file mode 100644 index 05babbc0f..000000000 --- a/js/smoke/scenarios/deno-node/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -.PHONY: setup test - -setup: - @echo "==> Setting up deno-node scenario" - mise install - @# Deno uses workspace links - need SDK dist/ folder to exist - if [ -n "$(BRAINTRUST_TAR)" ]; then \ - echo "==> Extracting SDK tarball for workspace link"; \ - TARBALL_PATH="$$(cd ../.. && pwd)/$(BRAINTRUST_TAR)"; \ - echo "==> Resolved tarball path: $$TARBALL_PATH"; \ - cd ../../.. && tar -xzf "$$TARBALL_PATH" --strip-components=1 package/dist; \ - else \ - echo "==> Building SDK for workspace link"; \ - cd ../../.. && pnpm exec turbo build --filter=braintrust; \ - fi; \ - -test: setup - @# Run only tests under the local tests/ directory - mise exec -- deno test --sloppy-imports --allow-all tests/*.test.ts diff --git a/js/smoke/scenarios/deno-node/README.md b/js/smoke/scenarios/deno-node/README.md deleted file mode 100644 index 5e369e395..000000000 --- a/js/smoke/scenarios/deno-node/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Deno Node Build Test - -Tests the main Node.js-compatible build (`braintrust` package) in Deno runtime. - -## Design Decisions - -### Local Package Linking - -Uses Deno's `links` feature in `deno.json` to test against local SDK builds: - -- `"links": ["../../..", "../../shared"]` ensures we use the workspace versions -- No need to publish or pack - Deno reads from `../../../dist/` directly - -### Sloppy Imports - -Requires `--sloppy-imports` flag because the shared test package uses extensionless imports: - -- Shared package: `import { foo } from "./helpers/types"` (no `.ts`) -- This is standard for TypeScript/Node.js but requires `--sloppy-imports` in Deno -- Alternative would be adding `.ts` extensions, but that may break Node.js tooling - -### npm Compatibility - -Uses `nodeModulesDir: "auto"` to enable Deno's npm package resolution: - -- Allows `npm:braintrust@^2.0.2` imports -- Combined with `links`, resolves to local workspace packages diff --git a/js/smoke/scenarios/deno-node/deno.json b/js/smoke/scenarios/deno-node/deno.json deleted file mode 100644 index 4e38fc8f6..000000000 --- a/js/smoke/scenarios/deno-node/deno.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "imports": { - "@std/assert": "jsr:@std/assert@^1.0.14", - "@braintrust/smoke-test-shared": "jsr:@braintrust/smoke-test-shared", - "braintrust": "npm:braintrust" - }, - "nodeModulesDir": "auto", - "links": ["../../..", "../../shared"] -} diff --git a/js/smoke/scenarios/deno-node/mise.toml b/js/smoke/scenarios/deno-node/mise.toml deleted file mode 100644 index dd563ce6f..000000000 --- a/js/smoke/scenarios/deno-node/mise.toml +++ /dev/null @@ -1,3 +0,0 @@ -[tools] -deno = "latest" -pnpm = "10.26.2" diff --git a/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts b/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts deleted file mode 100644 index cd70263a0..000000000 --- a/js/smoke/scenarios/deno-node/tests/shared-suite.test.ts +++ /dev/null @@ -1,75 +0,0 @@ -// @ts-nocheck -/** - * Deno smoke test using shared test suites - */ - -import { assertEquals } from "@std/assert"; -import { - runTests, - expectFailure, - testBasicSpanLogging, - testMultipleSpans, - testDirectLogging, - testJSONAttachment, - testAsyncLocalStorageTraced, - testNestedTraced, - testCurrentSpan, - testCoreLoggingExports, - testDatasetExports, - testPromptExports, - testExperimentExports, - testEvalExports, - testTracingExports, - testClientWrapperExports, - testUtilityExports, - testFunctionExports, - testFramework2Exports, - testIDGeneratorExports, - testTestingExports, - testStateManagementExports, - testBuildResolution, - testMustacheTemplate, - testNunjucksTemplate, - testEvalSmoke, -} from "@braintrust/smoke-test-shared"; -import * as braintrust from "braintrust"; - -Deno.test("Run shared test suites", async () => { - const { failed } = await runTests({ - name: "deno-node", - braintrust, - tests: [ - testCoreLoggingExports, - testDatasetExports, - testPromptExports, - testExperimentExports, - testEvalExports, - testTracingExports, - testClientWrapperExports, - testUtilityExports, - testFunctionExports, - testFramework2Exports, - testIDGeneratorExports, - testTestingExports, - testStateManagementExports, - testBuildResolution("node"), - testBasicSpanLogging, - testMultipleSpans, - testDirectLogging, - testJSONAttachment, - testAsyncLocalStorageTraced, - testNestedTraced, - testCurrentSpan, - testEvalSmoke, - testMustacheTemplate, - expectFailure( - testNunjucksTemplate, - (e: { message: string }) => - e.message.includes("requires @braintrust/template-nunjucks"), - "Nunjucks requires separate package", - ), - ], - }); - - assertEquals(failed.length, 0, "All tests should pass"); -}); diff --git a/js/smoke/shared/README.md b/js/smoke/shared/README.md index 1cac4d920..767eb526a 100644 --- a/js/smoke/shared/README.md +++ b/js/smoke/shared/README.md @@ -563,9 +563,9 @@ export const testSomething = register("testSomething", async (braintrust) => { 1. Make changes to test suites in `src/` 2. Run `npm run build` to rebuild -3. Test in target environment (e.g., Deno): +3. Test in a target smoke scenario: ```bash - cd ../scenarios/deno-node + cd ../scenarios/ make test ``` 4. Verify tests pass in multiple environments diff --git a/js/smoke/tests/deno/deno.json b/js/smoke/tests/deno/deno.json deleted file mode 100644 index f465329bd..000000000 --- a/js/smoke/tests/deno/deno.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "npmDependencies": { - "@next/env": "14.2.3", - "eventsource-parser": "1.1.2", - "uuid": "9.0.1", - "dotenv": "16.4.5", - "argparse": "2.0.1", - "chalk": "4.1.2" - }, - "workspace": { - "members": [] - }, - "imports": { - "@vercel/functions": "npm:@vercel/functions@^3.3.0", - "ajv": "npm:ajv@^8.17.1", - "eventsource-parser": "npm:eventsource-parser@^3.0.6", - "mustache": "npm:mustache@^4.2.0", - "uuid": "npm:uuid@^13.0.0", - "zod": "npm:zod@^4.1.12", - "nunjucks": "npm:nunjucks@^3.2.4", - "zod-to-json-schema": "npm:zod-to-json-schema@^3.25.0" - }, - "tasks": { - "test:shared": "deno test --config ./deno.json --allow-env --allow-read=.,$BRAINTRUST_BUILD_DIR --allow-net shared_suite_test.ts", - "test:legacy": "deno test --config ./deno.json --allow-env --allow-read=.,$BRAINTRUST_BUILD_DIR --allow-net span_test.ts" - } -}