TypeScript SDK Reference

​

Module Exports

// Values
export { Bitfab, BitfabError, BitfabFunction, DbBranchReplayError, MixedTracingError, ReplayError, NO_MOCK_OVERRIDE, getCurrentReplayBranch, getCurrentSpan, getCurrentTrace } from "@bitfab/sdk"
export { BitfabClaudeAgentHandler, BitfabLangChainCallbackHandler, BitfabLangGraphCallbackHandler, BitfabLangGraphIntegration, BitfabOpenAIAgentHandler, BitfabOpenAITracingProcessor, BitfabRivetIntegration, BitfabVercelAiHandler } from "@bitfab/sdk"
export { DatasetsClient, HttpClient, SpansClient, TracesClient, VerdictsClient, defineReplayRegistry, finalizers, flushTraces, reportReplayProgress, seedFromRegistry, serializeReplayResult } from "@bitfab/sdk"
export { __version__, DEFAULT_SERVICE_URL, BITFAB_PROGRESS_PREFIX, SUPPORTED_PROVIDERS } from "@bitfab/sdk"

// Types
export type {
  ActiveSpanContext,
  AdaptContext,
  BitfabAgentMiddleware,
  BitfabLangGraphAgent,
  AdaptInputsFn,
  AddDatasetGradersResult,
  AddDatasetTracesResult,
  AllowedEnvVars,
  ArchiveAssertionsParams,
  BamlExecutionResult,
  BitfabConfig,
  BitfabLanguageModelMiddleware,
  CaptureSurface,
  CaptureWhen,
  CapturedSpan,
  CodeChangeFile,
  CurrentSpan,
  CurrentTrace,
  Dataset,
  DatasetGraderRef,
  DatasetTraceIds,
  DbBranchOptions,
  DbBranchTimings,
  DbSnapshotConfig,
  DbSnapshotProvider,
  DbSnapshotRef,
  DetachedTrace,
  GraderRerun,
  GraderRerunProgress,
  GraderRerunResult,
  GraderRerunStatus,
  LangGraphIntegrationOptions,
  ListDatasetsParams,
  MockOverride,
  MockOverrideCtx,
  MockOverrideInput,
  MockOverrideResolver,
  MockStrategy,
  MockValue,
  NodeMatcher,
  NodeMethodDecorator,
  NodeOptions,
  ProviderDefinition,
  RemoveDatasetGradersResult,
  RemoveDatasetTracesResult,
  ReplayBranch,
  ReplayInterrupt,
  ReplayInterruptSignal,
  ReplayItem,
  ReplayItemFinishProgress,
  ReplayItemStartProgress,
  ReplayOptions,
  ReplayOptionsFactory,
  ReplayProgress,
  ReplayProgressItem,
  ReplayRegistration,
  ReplayRegistry,
  ReplayRegistryContext,
  ReplayRegistryOptions,
  ReplayResult,
  RerunGradersOptions,
  RerunGradersResult,
  RivetContextLike,
  RivetEffectType,
  RivetIntegrationOptions,
  SaveAssertion,
  SaveAssertionsAllParams,
  SaveAssertionsParams,
  SaveDatasetParams,
  SaveDatasetResult,
  SaveScenarioActualOutcome,
  SaveScenarioData,
  SaveScenarioHistoryEntry,
  SaveScenarioParams,
  Scenario,
  ScenarioAction,
  ScenarioActualOutcome,
  ScenarioData,
  ScenarioDataAppend,
  ScenarioDataOrigin,
  ScenarioHistoryEntry,
  ScenarioSource,
  ScenarioTrigger,
  SeedCase,
  SeedCaseOptions,
  SeedResult,
  SeedRunOptions,
  SpanLookup,
  SpanMethodDecorator,
  SpanMethodDecoratorContext,
  SpanNodeMeta,
  SpanOccurrence,
  SpanOptions,
  SpanType,
  TokenUsage,
  TraceIngestionType,
  TraceAssertion,
  TraceAssertionsResult,
  TraceAssertionSource,
  TraceAssertionsUpdate,
  TraceResponse,
  TraceTarget,
  TraceTargetOccurrence,
  TracingProcessor,
  VercelCallParams,
  VercelGenerateResult,
  VercelStreamResult,
  VerdictAction,
  VerdictConfidence,
  VerdictOutcome,
  VerdictUpdate,
  WrapBAMLOptions,
  WrappedBamlFn,
} from "@bitfab/sdk"

​

Constants

​

class Bitfab

​

new Bitfab(config: BitfabConfig)

​

captureEnabled

client.captureEnabled: boolean

​

span / withSpan

withSpan<TArgs extends unknown[], TReturn>(
  traceFunctionKey: string,
  optionsOrFn: SpanOptions | ((...args: TArgs) => TReturn),
  maybeFn?: (...args: TArgs) => TReturn,
): (...args: TArgs) => TReturn

• Returns a wrapper whether or not capture is on. With captureEnabled: false, the wrapper runs the function untraced. It sends nothing, except inside a replay item or the one call seedTrace runs
• Span name defaults to the function’s qualified name, resolved per call: Order.process when the function runs as a method of an Order instance (or Order.build for a static method), process for a plain function, then traceFunctionKey when the function is anonymous. The raw fn.name still travels separately as function_name
• Span type defaults to "custom"
• experimentId links the span to an experiment. If the span starts a trace, that trace is linked too. testRunId is a deprecated alias with the same meaning, and passing both with different values throws
• captureWhen: "nested" records the span only when another Bitfab span is active. Without a parent, the wrapped function runs normally. It does not create a root trace. The default is "always". An unknown value warns once and falls back to that default
• Input arguments are serialized via superjson. This preserves type information
• The return value is serialized as output, whether it is a plain value or a resolved Promise
• A value that fails to serialize is stubbed instead of dropping the span. A warning notes it may not be replayable. A span whose encoded payload still exceeds the per-request byte ceiling ships anyway, with its largest fields stubbed first and its identifying fields stubbed last. This way an oversized span degrades instead of vanishing
• SpanOptions.finalize?: (result) => unknown | Promise<unknown> records a serializable view of a non-serializable result, such as a live stream. The raw result is returned to the caller unchanged. await finalize(result) is recorded as the span output instead. It runs in the background. A throwing finalize records an error instead of crashing. For an async-generator result, it receives the array of yielded values once iteration completes, and its return is recorded instead of { yielded, return }. Pair it with the exported finalizers.aiSdk for the Vercel AI SDK, or finalizers.readableStream
• An async-generator span remains open until iteration completes, returns early, or throws. A nested span created inside the generator body inherits that span. To include spans created by the consumer in the same trace, wrap the controller that owns the iteration in an outer root span
• A thrown error is recorded on the span, in the error and error_source: "code" fields. It is then re-thrown
• Spans nest automatically, via AsyncLocalStorage on Node or a module-level stack as a browser fallback
• Spans exist only where you create them. Nesting is automatic, but only between spans that exist. One wrapper around the outermost function records a single-node trace. See Instrumentation
• withSpan() is the opt-in tracing surface. trace()/withTrace() and node()/withNode() are the opt-out surface. A withSpan() entered beneath an active subtree trace throws MixedTracingError. Configure a discovered call with node()/withNode() instead
• The browser fallback does not isolate concurrent async chains. Promise.all with independent withSpan calls may see the wrong parent

span(
  traceFunctionKey: string,
  options?: SpanOptions,
): SpanMethodDecorator

• Requires TypeScript 5.0 or newer, and a compiler or transpiler that supports the standard ECMAScript decorator transform
• experimentalDecorators must be disabled or omitted. The legacy decorator transform is not supported
• emitDecoratorMetadata must be disabled or omitted. It is not compatible with standard decorators
• Supports instance, static, and private methods
• Preserves the method receiver, arguments, return type, errors, nesting, and all SpanOptions
• Within opt-in tracing, prefer withSpan to the optional span() decorator because it supports every TypeScript version and callable shape. For new workflows on a supported build path, start with opt-out withTrace / trace and configure discovered calls with withNode / node

​

trace / withTrace

trace(traceFunctionKey: string, options?: TraceOptions): TraceMethodDecorator

withTrace<TArgs extends unknown[], TReturn>(
  traceFunctionKey: string,
  fn: (...args: TArgs) => TReturn,
): (...args: TArgs) => TReturn

withTrace<TArgs extends unknown[], TReturn>(
  traceFunctionKey: string,
  options: TraceOptions,
  fn: (...args: TArgs) => TReturn,
): (...args: TArgs) => TReturn

interface TraceOptions {
  name?: string
  type?: SpanType
  mockOnReplayDefault?: boolean
  maxDepth?: number
  maxCapturedSubtreeSpans?: number
  exclude?: readonly string[] | ReadonlySet<string>
  includeWrappers?: boolean
}

​

node / withNode

node(options?: NodeOptions): NodeMethodDecorator

• Outside an active subtree trace, the method runs normally with no capture or replay behavior. Beneath an active withSpan() with no enclosing trace, it throws MixedTracingError instead. node() belongs to the opt-out surface, so it has no trace to configure there
• capture: true (the default) keeps the call captured, with its inputs, output, and errors
• name, type, experimentId, mockOnReplay, and finalize have the same behavior as their SpanOptions counterparts
• Under a trace with mockOnReplayDefault: true, an omitted node policy inherits the trace default. Setting mockOnReplay: false on the node keeps it live instead, under the default "marked" replay strategy
• capture: false omits the call. Its captured descendants attach to the nearest captured parent instead
• capture: false with mockOnReplay: true throws BitfabError, because an uncaptured call has no recorded output
• Requires a compatible @bitfab/sdk/transform integration to discover the method. Standard and legacy method decorator output are accepted when the transform runs before decorators are lowered

withNode<This, TArgs extends unknown[], TReturn>(
  optionsOrFn: NodeOptions | ((this: This, ...args: TArgs) => TReturn),
  maybeFn?: (this: This, ...args: TArgs) => TReturn,
): (this: This, ...args: TArgs) => TReturn

​

getFunction

getFunction(traceFunctionKey: string): BitfabFunction

​

wrapBAML

// Form 1: uses bamlClient from constructor
wrapBAML<TArgs, TReturn>(
  method: (...args: TArgs) => Promise<TReturn>,
  options?: WrapBAMLOptions,
): WrappedBamlFn<TArgs, TReturn>

// Form 2: explicit client
wrapBAML<TArgs, TReturn>(
  bamlClient: unknown,
  method: (...args: TArgs) => Promise<TReturn>,
  options?: WrapBAMLOptions,
): WrappedBamlFn<TArgs, TReturn>

• BitfabError if form 1 is used without bamlClient in constructor
• BitfabError if the method has no .name

​

wrapJev

wrapJev<TArgs, TReply>(
  method: (...args: TArgs) => TReply | Promise<TReply>,
  options?: WrapJevOptions<TReply>,
): (...args: TArgs) => Promise<TReply>

• BitfabError if the method has no .name (bind the client’s method: jev.systemOne.bind(jev))

• If @boundaryml/baml cannot be loaded, the method is called directly, onCollector is not called, .collector is null, and a one-time [bitfab] warning is logged. In bundled server code (Next.js with Turbopack, for example), import "@bitfab/sdk/baml" once so the SDK can load it
• Otherwise, it creates a BAML Collector, calls the method through a tracked client, then:
  • Calls getCurrentSpan().setPrompt(...) with the rendered messages as JSON
  • Calls getCurrentSpan().addContext({ model, provider, inputTokens, outputTokens, durationMs })
• The onCollector callback fires after each invocation. Errors in the callback are swallowed
• Beneath a withTrace() / trace() root the BAML call is already recorded as its own llm span, so the prompt and metadata land on that span and the enclosing span is left alone. Passing the wrapper to withNode() configures that one span. See BAML under a trace root

​

wrapOpenRouter

wrapOpenRouter<TArgs, TResponse>(
  method: (...args: TArgs) => TResponse | Promise<TResponse>,
  options?: WrapOpenRouterOptions<TResponse>,
): (...args: TArgs) => Promise<TResponse>

• BitfabError if the method has no .name (bind the client’s method: openai.chat.completions.create.bind(openai.chat.completions))

​

wrapAnthropic

wrapAnthropic<TArgs, TResponse>(
  method: (...args: TArgs) => TResponse | Promise<TResponse>,
  options?: WrapAnthropicOptions<TResponse>,
): (...args: TArgs) => Promise<TResponse>

• BitfabError if the method has no .name (bind the client’s method: anthropic.messages.create.bind(anthropic.messages))

​

wrapOpenAIResponses

wrapOpenAIResponses<TArgs, TResponse>(
  method: (...args: TArgs) => TResponse | Promise<TResponse>,
  options?: WrapOpenAIResponsesOptions<TResponse>,
): (...args: TArgs) => Promise<TResponse>

• BitfabError if the method has no .name (bind the client’s method: openai.responses.create.bind(openai.responses))

​

getTrace

getTrace(traceId: string): DetachedTrace

• All methods on the returned handle block (return Promise<void>), resolving once the server has applied the change
• When capture is off outside a replay item or a seedTrace call, methods return Promise.resolve() immediately
• The server returns 404 if no trace exists with that ID. The returned promise rejects with a BitfabError in that case, rather than logging it

​

getTraceSpan

getTraceSpan(traceId: string, lookup: SpanLookup): Promise<CapturedSpan | null>

const firstModelCall = await client.getTraceSpan(traceId, {
  type: "llm",
  occurrence: "first",
})

​

getSpan

getSpan(spanId: string): Promise<CapturedSpan | null>

​

datasets

client.datasets.save(params: SaveDatasetParams): Promise<SaveDatasetResult>
client.datasets.list(params?: ListDatasetsParams): Promise<Dataset[]>
client.datasets.get(datasetId: string): Promise<Dataset>
client.datasets.update(datasetId: string, params: UpdateDatasetParams): Promise<Dataset>
client.datasets.listTraces(datasetId: string): Promise<DatasetTraceIds>
client.datasets.addTraces(datasetId: string, traceIds: string[]): Promise<AddDatasetTracesResult>
client.datasets.removeTraces(datasetId: string, traceIds: string[]): Promise<RemoveDatasetTracesResult>
client.datasets.addGraders(datasetId: string, graderIds: string[]): Promise<AddDatasetGradersResult>
client.datasets.removeGraders(datasetId: string, graderIds: string[]): Promise<RemoveDatasetGradersResult>
client.datasets.rerunGraders(datasetId: string, options?: RerunGradersOptions): Promise<RerunGradersResult>
client.datasets.getGraderRerun(datasetId: string, runId?: string): Promise<GraderRerun | null>

interface Dataset {
  id: string
  traceFunctionKey: string
  name: string
  description: string | null
  traceCount: number
  graders: { id: string; name: string | null }[]
  golden: boolean
  createdAt: string
  updatedAt: string
}

interface SaveDatasetParams {
  traceFunctionKey: string
  name: string
  description?: string
  golden?: boolean
}

interface SaveDatasetResult { dataset: Dataset; created: boolean }
interface ListDatasetsParams { traceFunctionKey?: string }
interface DatasetTraceIds { datasetId: string; traceIds: string[] }

interface AddDatasetTracesResult {
  dataset: Dataset
  addedTraceIds: string[]
  alreadyPresentTraceIds: string[]
  skippedTraceIds: string[]
}

interface RemoveDatasetTracesResult {
  dataset: Dataset
  removedTraceIds: string[]
  notPresentTraceIds: string[]
}

interface AddDatasetGradersResult {
  dataset: Dataset
  addedGraderIds: string[]
  alreadyAssignedGraderIds: string[]
  skippedGraderIds: string[]
}

interface RemoveDatasetGradersResult {
  dataset: Dataset
  removedGraderIds: string[]
  notAssignedGraderIds: string[]
}

type GraderRerunStatus = "pending" | "running" | "completed" | "errored"

interface GraderRerun {
  id: string
  status: GraderRerunStatus
  graderIds: string[]
  progress: { completedTraces: number; totalTraces: number; graderCount: number } | null
  result: { tracesGraded: number; gradersRun: number } | null
  error: string | null
  createdAt: string
  updatedAt: string
}

interface RerunGradersOptions {
  graderIds?: string[]
  timeoutMs?: number
}

interface RerunGradersResult {
  run: GraderRerun
  joinedExisting: boolean
}

• save is an upsert keyed on (traceFunctionKey, name). created is true for a new dataset and false when an existing one was updated. An omitted description leaves the existing one untouched.
• list takes an optional traceFunctionKey. Without it, every dataset in the organization is returned.
• update renames a dataset or changes its description in place and resolves to the updated dataset, which keeps its id, traces, and graders. UpdateDatasetParams is { name?: string, description?: string, golden?: boolean }. An omitted field is left untouched and an empty description clears it. Passing none of them rejects with BitfabError before any request. A blank name, or a name another dataset under the same trace function already uses, rejects with a 400 BitfabError.
• Dataset carries id, traceFunctionKey, name, description, traceCount, graders ({ id, name }[]), golden, createdAt, and updatedAt.
• golden: true on save or update marks the dataset as golden for the whole organization, a trusted reference set that is listed first in the app, in list_datasets, and in list. golden: false unmarks it, and omitting it leaves it as it is. Every dataset carries golden, read as false from an older server.
• listTraces returns { datasetId, traceIds }, the same membership a replay with datasetId selects.
• addTraces and addGraders accept 1 to 100 ids. They report partial acceptance instead of rejecting the whole call. An id outside the organization, or under another trace function, comes back in skippedTraceIds or skippedGraderIds. An id already present comes back in alreadyPresentTraceIds or alreadyAssignedGraderIds.
• removeTraces never deletes a trace, only its membership in the dataset. Ids that were not members come back in notPresentTraceIds. removeGraders reports notAssignedGraderIds the same way.
• rerunGraders re-scores every trace in the dataset. graderIds defaults to every assigned grader. An unassigned id rejects the call. The promise resolves once the run completes. Hold it and await it when you need the result, since nothing blocks while it waits. It rejects with GraderRerunError when the run errored, carrying the run, and with GraderRerunTimeoutError when the run is still going after timeoutMs (default 90,000). A timeout does not stop the run on Bitfab, so read it later with getGraderRerun. A request matching an in-flight run joins it, reported as joinedExisting: true.
• GraderRerun has status (pending | running | completed | errored), graderIds, progress ({ completedTraces, totalTraces, graderCount } while running), result ({ tracesGraded, gradersRun } when completed), and error.
• A dataset id from another organization rejects with a 404 BitfabError.

​

traces

client.traces.search(params: TraceSearchParams): Promise<TraceSearchResult>
client.traces.getAssertions(traceId: string): Promise<TraceAssertionsResult>
client.traces.saveAssertions(params: SaveAssertionsParams): Promise<SavedTraceAssertion[]>
client.traces.saveAssertionsAll(params: SaveAssertionsAllParams): Promise<SavedTraceAssertion[]>
client.traces.archiveAssertions(params: ArchiveAssertionsParams): Promise<string[]>
client.traces.generateAssertions(traceId: string, options?: GenerateAssertionsOptions): Promise<TraceAssertion[]>
client.traces.getScenario(traceId: string): Promise<Scenario>
client.traces.saveScenario(params: SaveScenarioParams): Promise<Scenario>

client.traces.search({
  callerMetadata?: Record<string, string>,
  name?: string,
  nameContains?: string,
  experimentId?: string,
  traceFunctionKey?: string,
  cursor?: string | null,
  limit?: number,
}): Promise<TraceSearchResult>

• getAssertions returns { assertions, inheritedFrom, generation }. generation reports the trace’s latest assertion generation, or null when generation never ran on that trace. Attach assertions to the ORIGINAL trace. Reading a replay that has none of its own returns the nearest ancestor’s assertions instead, resolved through the replay lineage. That ancestor trace is named in inheritedFrom. inheritedFrom is null when the assertions are the trace’s own. Writing assertions through a replay trace id is refused, and the error names the original to retry with.
• The common case is to omit targetOnEvaluatedTrace. Omitting it checks the whole trace. Set it to narrow the check instead, to either { kind: "output" } or { kind: "span", name, occurrence? }.
• Targets are span names, never span ids, because an id captured on the original resolves to nothing on the replay. occurrence accepts "first", "last" (the default), or a 0-based index for a span the trace calls more than once.
• assertion, passCriteria, and failCriteria are the same three fields save_grader takes, so an assertion that proves out across many traces is promoted into a grader by copying them.
• humanNote is people-only context attached to the assertion. Pass it on a save to write it, omit it on an edit to preserve what is stored, and pass null to clear it. It must never be used as evidence when judging a replay.
• Passing an entry’s id edits that assertion. Omitting it adds a new one instead. This way two callers adding different assertions to one trace never overwrite each other. archiveAssertions hides rows from every read. It keeps them for audit.
• saveAssertions’ source defaults to "agent" when omitted, recording that a coding agent, not a person, authored the assertion.
• justification is the evidence for the assertion: one entry per span you relied on, each { spanId, text }, so the reason sits on the span a reviewer can open rather than in one paragraph over all of them. Cite spans on the assertion’s own trace. categoryJustification does the same for category_id, saying why that category fits. Omit either on an edit to preserve it. Changing the category clears categoryJustification, since it justified the old one.
• assigneeEmail names the organization member who should review the assertion. The server resolves the address against that organization’s members and refuses one belonging to nobody in it, so a typo fails the call instead of quietly leaving the assertion unrouted. Omit it on an edit to preserve the current assignee, and pass null to leave it to nobody. Reads carry a nullable assignee with id, fullName, email, and imageUrl. Assigning routes an assertion rather than deciding it, so it never returns the assertion to "pending" the way an edit to the wording, criteria, target, or category does.
• Reads carry one review decision in approvalState, approvedBy, and approvedAt. It approves both whether the assertion applies to its trace and whether its current category assignment is correct, using the two separate justifications as evidence. Only a person in Bitfab sets it. Editing assertion content or changing its category resets the decision.
• The save returns once the text is saved as sent. Bitfab then checks each entry that carries assertion text against the writing rules in the background and, when the text breaks one, rewrites it in place as one well-formed assertion, usually within a minute. Each returned assertion carries rewrite, whose status is pending for an entry being checked and not_checked for one that is not. Read the result later with getAssertions: each row carries its latest rewrite, with status (pending, rewritten, unchanged, or check_failed, which also covers a check unfinished 15 minutes after the save), sentAssertion, changes, separateAssertions, unresolved, and error, or null when no check ever ran on it. separateAssertions holds other claims the sent text bundled. They were not saved, so save each as its own assertion. Pass skipGuidedRewrite: true to save text exactly as sent, with no check.
• saveAssertions writes one trace. saveAssertionsAll takes one update per trace and writes them all in a single request, so a publisher covering hundreds of traces makes one call. The singular delegates to it, so both go through the same route.
• The server writes the whole batch in one transaction, so a rejected batch writes nothing and there is no half-saved state to reconcile. Results come back as one flat array covering every trace, each row carrying its own traceId. A batch takes up to 500 traces, up to 50 assertions per trace, and at most 1000 assertions in total. Passing an empty updates array writes nothing and sends no request.

await client.traces.saveAssertionsAll({
  updates: traceIds.map((traceId) => ({
    traceId,
    assertions: [
      {
        assertion: "Departs before 9am",
        justification: [
          { spanId: departureSpanId, text: "Booked the 06:15 departure" },
        ],
      },
    ],
  })),
})

​

Generate assertions

client.traces.generateAssertions(traceId, {
  guidance?: string,
  categoryIds?: string[],
  regenerate?: boolean,
  timeoutMs?: number,
}): Promise<TraceAssertion[]>

• guidance is free text that steers what the assertions are about, such as a root cause analysis of why the trace went wrong. It narrows the evidence and produces smaller assertions aimed at that failure. Up to 10,000 characters.
• categoryIds limits generation to those categories. Omit it to let Bitfab choose from every category in the organization. An id that is not one of the organization’s categories fails the call before any work starts.
• A trace keeps one set of generated drafts. Calling again returns those drafts instead of generating twice, even when the new call passes different guidance or categoryIds. Pass regenerate: true to start over. The previous generated drafts are archived once the new ones are saved. Assertions a person wrote are never replaced.
• A trace runs one generation at a time. Calling generateAssertions while one is running joins it and never starts a second one. Passing regenerate: true while one is running joins it too.
• The promise resolves with the drafts once generation finishes. Hold it and await it when you need the drafts, or start several traces at once and Promise.all them. It rejects with AssertionGenerationError when generation failed, carrying traceId. Pass regenerate: true to try again. It rejects with AssertionGenerationTimeoutError when generation has not finished within timeoutMs (default 600,000). A timeout does not stop generation on Bitfab.
• To check on a generation from anywhere, call getAssertions and read generation. It carries status ("running", "completed", or "failed"), error, assertionIds (the drafts the latest completed generation saved, in order), startedAt, and finishedAt. It is null when generation never ran on that trace.
• A replay trace id is refused, and the error names the original to retry with.

const generation = client.traces.generateAssertions(traceId, {
  guidance: "The agent booked a flight after the user asked for a train.",
})

// other work runs while Bitfab generates

const drafts = await generation
console.log(drafts.map((a) => a.assertion))

type TraceTargetOccurrence = "first" | "last" | number
type TraceTarget =
  | { kind: "output" }
  | { kind: "span"; name: string; occurrence?: TraceTargetOccurrence }
type TraceAssertionSource = "human" | "agent"

type SaveAssertion = {
  category_id?: string | null
  id?: string
  assertion: string
  humanNote?: string | null
  passCriteria?: string | null
  failCriteria?: string | null
  targetOnEvaluatedTrace?: TraceTarget | null
  assigneeEmail?: string | null
}

type SaveAssertionsParams = {
  traceId: string
  assertions: SaveAssertion[]
  source?: TraceAssertionSource
  skipGuidedRewrite?: boolean
}

type TraceAssertionsUpdate = {
  traceId: string
  assertions: SaveAssertion[]
}

type SaveAssertionsAllParams = {
  updates: TraceAssertionsUpdate[]
  source?: TraceAssertionSource
  skipGuidedRewrite?: boolean
}

type AssertionRewrite = {
  status: "pending" | "rewritten" | "unchanged" | "not_checked" | "check_failed"
  sentAssertion: string | null
  changes: string[]
  separateAssertions: string[]
  unresolved: string[]
  error: string | null
}

type SavedTraceAssertion = TraceAssertion & { rewrite: AssertionRewrite }

type ArchiveAssertionsParams = {
  traceId: string
  assertionIds: string[]
}

type TraceAssertion = {
  category_id: string | null
  category: CategorySummary | null
  id: string
  traceId: string
  assertion: string
  humanNote: string | null
  passCriteria: string | null
  failCriteria: string | null
  targetOnEvaluatedTrace: TraceTarget | null
  source: TraceAssertionSource
  assignee: Assignee | null
  createdAt: string
  updatedAt: string
}

type AssertionGenerationStatus = "running" | "completed" | "failed"

type AssertionGeneration = {
  status: AssertionGenerationStatus
  error: string | null
  assertionIds: string[]
  startedAt: string
  finishedAt: string | null
}

type TraceAssertionsResult = {
  assertions: (TraceAssertion & { rewrite: AssertionRewrite | null })[]
  inheritedFrom: string | null
  generation: AssertionGeneration | null
}

​

Read and update a scenario

client.traces.getScenario(traceId: string): Promise<Scenario>

client.traces.saveScenario({
  traceId: string,
  name?: string,
  description?: string,
  trigger?: ScenarioTrigger,
  data?: SaveScenarioData,
  appendData?: ScenarioDataAppend,
  actualOutcome?: SaveScenarioActualOutcome | null,
  categoryIds?: string[],
}): Promise<Scenario>

• name, description, trigger, and data each replace the saved value whole. Fields left out are kept.
• appendData adds to the saved data without resending it: its history entries go after the saved ones and its missing items after the saved missing items. Appends made at the same time by different callers all land.
• Send data or appendData, not both. Passing both, or nothing besides traceId, is a type error.
• actualOutcome is what the run actually did: a short summary and the actions it took, each with its arguments in details, what came back in result, whether it failed, and when it happened in at. It is one example of a response, never what the run should have done. Bitfab never fills it in, so it is only ever what a caller saved. It replaces the saved value whole, null clears it, and leaving it out keeps it. A save that sends only actualOutcome, null included, is valid.
• actualOutcome.source names the span the outcome was read from, on a trace linked to the scenario. Leave it out or set it to null when it was not read from one. Every scenario read back carries actualOutcome, null when none was saved.
• categoryIds files the scenario under any number of categories, by id from client.categories. It replaces the whole set: leave it out to keep the current set, and pass an empty array to clear it. An id outside the API key’s organization fails the save with a 400 BitfabError naming the missing ids. Every scenario read back carries categories, each a CategorySummary, sorted by title.
• A history entry’s source names the span its text came from, on this trace or another trace linked to the same scenario. Leave it out or set it to null when the text is not in such a trace. A source naming a span on any other trace is refused.
• The server rebuilds triggerSummary and dataSummary from what is saved, and the call resolves with the saved scenario.
• saveScenario only updates a scenario that already exists. A trace with no scenario rejects with a 404 BitfabError, from either method.

await client.traces.saveScenario({
  traceId,
  appendData: {
    missing: ["The traveler's loyalty program number"],
  },
})

await client.traces.saveScenario({
  traceId,
  actualOutcome: {
    summary: "The assistant texted the traveler to confirm the new flight.",
    actions: [
      {
        name: "send_sms_message",
        details: { to: "+15555550100", body: "You're on the 9am flight." },
        result: "delivered",
        failed: false,
        at: "2026-09-01T09:05:00Z",
      },
    ],
  },
})

await client.traces.saveScenario({
  traceId,
  categoryIds: [categoryId],
})

type ScenarioSource = {
  spanId: string
  sourceField: "input" | "output"
  text: string
}

type ScenarioTrigger = {
  inputId: number | null
  inputType: string
  channel: string
  from: string
  to: string | null
  cc: string | null
  subject: string | null
  type: string
  eventTime: string
  eventTimeSource: "message date" | "run start"
  source: "request" | "run instructions"
  content: string
  contentPassages: string[]
  contentIsWholeMessage: boolean
  sources: ScenarioSource[]
}

type ScenarioDataOrigin = {
  section: string
  channel: string | null
  from: string | null
  to: string | null
  cc: string | null
  subject: string | null
  sentAtText: string | null
}

type ScenarioHistoryEntry = {
  role: "thread start" | "fact"
  kind: string
  quote: string
  summary: string | null
  origin: ScenarioDataOrigin | null
  sentAt: string | null
  weekday: string | null
  sincePrevious: string | null
  source: ScenarioSource | null
}

type ScenarioData = {
  hasHistory: boolean
  topic: string
  history: ScenarioHistoryEntry[]
  missing: string[]
}

type SaveScenarioHistoryEntry = Omit<ScenarioHistoryEntry, "source"> & {
  source?: ScenarioSource | null
}

type SaveScenarioData = Omit<ScenarioData, "history"> & {
  history: SaveScenarioHistoryEntry[]
}

type ScenarioDataAppend = {
  history?: SaveScenarioHistoryEntry[]
  missing?: string[]
}

type ScenarioAction = {
  name: string
  details: Record<string, unknown>
  result: string | null
  failed: boolean
  at: string | null
}

type ScenarioActualOutcome = {
  summary: string
  actions: ScenarioAction[]
  source: ScenarioSource | null
}

type SaveScenarioActualOutcome = Omit<ScenarioActualOutcome, "source"> & {
  source?: ScenarioSource | null
}

type Scenario = {
  id: string
  createdAt: string
  updatedAt: string
  organizationId: string
  traceFunctionId: string
  sourceTraceId: string
  extractor: "noah"
  name: string
  description: string
  trigger: ScenarioTrigger
  data: ScenarioData
  triggerSummary: string
  dataSummary: string
  actualOutcome: ScenarioActualOutcome | null
  categories: CategorySummary[]
}

​

spans

client.spans.list(traceId: string, params?: SpanListParams): Promise<SpanListResult>

type SpanListParams = {
  spanType?: "llm" | "agent" | "function" | "guardrail" | "handoff" | "custom"
  spanName?: string
  fields?: ("input" | "output" | "reasoning" | "content" | "errors" | "contexts")[]
  cursor?: string | null
  limit?: number // 1 to 200 (50 with fields), default 50
}

type SpanListResult = {
  spans: ListedSpan[]
  nextCursor: string | null
  hasMore: boolean
}

type ListedSpan = {
  id: string
  traceId: string
  parentSpanId: string | null
  name: string | null
  type: string | null
  startedAt: string | null
  endedAt: string | null
  input?: unknown // present only when requested in fields
  output?: unknown
  reasoning?: unknown
  content?: unknown
  errors?: unknown
  contexts?: unknown
}

let cursor: string | null = null
do {
  const page = await client.spans.list(traceId, { fields: ["input"], cursor })
  for (const span of page.spans) {
    console.log(span.name, span.input)
  }
  cursor = page.nextCursor
} while (cursor)

​

organizationMembers

client.organizationMembers.list(): Promise<OrganizationMember[]>

type OrganizationMember = {
  id: string
  fullName: string | null
  email: string | null
  imageUrl: string | null
}

​

categories

client.categories.save(params: SaveCategoryParams): Promise<Category>
client.categories.get(id: string): Promise<Category>
client.categories.list(): Promise<Category[]>
client.categories.delete(id: string): Promise<Category>

type SaveCategoryParams = {
  id?: string
  title: string
  description?: string
}

type JustificationSpan = { spanId: string; text: string }
type Justification = JustificationSpan[]

type CategorySummary = {
  id: string
  title: string
  description: string
}

type Category = CategorySummary & {
  organizationId: string
  createdAt: string
  updatedAt: string
}

const category = await client.categories.save({
  title: "Mandate compliance",
  description: "Checks explicit user instructions",
})
const [assertion] = await client.traces.saveAssertions({
  traceId,
  assertions: [{ assertion: "Escalates conflicting calendar requests", category_id: category.id }],
})
await client.traces.saveAssertions({
  traceId,
  assertions: [{ id: assertion.id, assertion: assertion.assertion, category_id: null }],
})
await client.categories.delete(category.id)

​

verdicts

client.verdicts.save(update: VerdictUpdate, experimentId?: string): Promise<VerdictOutcome>
client.verdicts.saveAll(updates: VerdictUpdate[], experimentId?: string): Promise<VerdictOutcome[]>
client.verdicts.skip(target: VerdictTarget & { annotation?: string }, experimentId?: string): Promise<VerdictOutcome>
client.verdicts.abstain(target: VerdictTarget & { annotation?: string }, experimentId?: string): Promise<VerdictOutcome>
client.verdicts.archive(target: ArchiveVerdictTarget, experimentId?: string): Promise<VerdictOutcome>
client.verdicts.saveHuman(update: HumanVerdictUpdate): Promise<HumanVerdictOutcome>
client.verdicts.saveHumanAll(updates: HumanVerdictUpdate[]): Promise<HumanVerdictOutcome[]>
client.verdicts.get(traceId: string): Promise<TraceVerdicts | null>
client.verdicts.getAll(traceIds: string[]): Promise<TraceVerdicts[]>
client.verdicts.generateEvidence(traceId: string, assertionId: string): Promise<AssertionVerdictEvidence[]>

• Key a direct verdict by traceId. Key a replay verdict by originalTraceId plus the experimentId it ran under, adding attempt when the experiment ran each trace more than once.
• verdict is "pass", "fail", or "abstained". A pass or fail takes a required annotation and optional confidence and evidence. An abstention takes an optional annotation.
• Every agent verdict scores one approved assertion, so save, saveAll, and skip require assertionId. Leaving it out is a type error, and at runtime the call throws BitfabError before any request. Verdicts are only written on assertions, never on a whole trace. archive is the one write that may omit it, which clears an old whole-trace agent verdict. saveHuman and saveHumanAll require it too, enforced the same way.
• skip(target, experimentId?) withholds a verdict. archive(target, experimentId?) clears a previous one. VerdictTarget takes either traceId or originalTraceId with optional attempt, plus a required assertionId. ArchiveVerdictTarget is the same with assertionId optional. Batch updates can still use { skip: true } and { archive: true }. Skip is the right answer for an attempt that crashed, was punctured, or fell back to a schema default. It is also right for an assertion whose target could not be resolved.
• A withheld verdict takes an annotation saying why, stored on it and returned by get and getAll. Without one, a skip reads the same whether nobody could decide or nobody looked, and a skip sent without an annotation clears the note the previous verdict carried.
• abstain(target, experimentId?) records an abstention, the same write as { verdict: "abstained" } in a batch. An abstention is the judge’s third answer: the evidence it read does not show the part of the run the assertion checks, so nothing can confirm or deny it. It counts as a fail in every pass rate, since anything short of an evidenced pass is a fail, but it is stored and read back apart from a fail, so a reader sees judge coverage rather than the agent’s behavior. It is not a skip: a skip is for a check that never ran, an abstention is for a check that ran and found nothing to judge. Pass annotation to say what the evidence would have had to show. It reads back as verdict: "abstained", and the trace’s abstained count says how many of its failed verdicts are abstentions.
• VerdictOutcome.key echoes the id you addressed the verdict by, so a caller can verify persistence without ever holding a server-generated replay trace id.
• get and getAll read by trace ID only. After writing a replay verdict, pass the returned VerdictOutcome.traceId to get to read it back.
• A verdict may carry evidence, with one { spanId, text } entry per cited span on the evaluated trace. Omit it to preserve saved evidence. Pass null to clear it. get and getAll return the evidence with each whole-trace and per-assertion verdict.
• generateEvidence generates assertion verdict evidence, the same call as the generate_verdict_evidence MCP tool. Pass the original or replay trace being judged; the assertion must belong to that trace or be inherited through its replay lineage. Each candidate comes from the selected trace and contains spanId, text, sourceField (the span field the quote was copied from), spanName, signature-only parameters (names and runtime types, never values), spanType, and isMocked. Save only the candidate’s { spanId, text } fields as verdict evidence; saved evidence remains on verdicts.get and verdicts.getAll.

type VerdictConfidence = "VeryLow" | "Low" | "Medium" | "High" | "VeryHigh"
type VerdictAction = "set" | "archived" | "no-active-verdict" | "skipped" | "abstained"
type Verdict = "pass" | "fail" | "abstained"
type PassFailVerdict = "pass" | "fail"

type VerdictAddress = { traceId: string } | { originalTraceId: string; attempt?: number }
type VerdictTarget = VerdictAddress & { assertionId: string }
type ArchiveVerdictTarget = VerdictAddress & { assertionId?: string }

type VerdictUpdate =
  | (VerdictTarget & (
      | { verdict: PassFailVerdict; annotation: string; confidence?: VerdictConfidence; evidence?: Justification | null }
      | { verdict: "abstained"; annotation?: string }
      | { skip: true; annotation?: string }
    ))
  | (ArchiveVerdictTarget & { archive: true })

type VerdictOutcome = {
  key: string
  traceId: string
  action: VerdictAction
}

type HumanVerdictUpdate = {
  traceId: string
  assertionId: string
  verdict: PassFailVerdict
  annotation: string
  confidence?: VerdictConfidence
  evidence?: Justification | null
}

type HumanVerdictOutcome = {
  traceId: string
  assertionId: string | null
  verdict: PassFailVerdict
  action: "set"
}

type VerdictStatus = "judged" | "skipped" | "not-judged"
type VerdictSource = "human" | "agent"

type AssertionVerdict = {
  assertionId: string
  assertion: string | null
  verdict: Verdict | null // "pass", "fail", or "abstained"
  verdictStatus: VerdictStatus
  source: VerdictSource
  annotation: string | null
  evidence: Justification | null
  confidence: VerdictConfidence | null
  approved: boolean
}

type TraceVerdicts = {
  traceId: string
  verdict: Verdict | null
  verdictStatus: VerdictStatus
  annotation: string | null
  evidence: Justification | null
  approved: boolean
  passed: number
  failed: number
  abstained: number // how many of the failed verdicts are abstentions
  assertions: AssertionVerdict[]
}

​

graders

client.graders.getVerdicts(params: GetGraderVerdictsParams): Promise<GraderVerdict[]>

type GetGraderVerdictsParams = {
  traceIds?: string[]
  graderId?: string
  limit?: number
}

type GraderVerdictSource = "human" | "live_grader"

type GraderVerdict = {
  traceId: string
  graderId: string
  graderName: string | null
  graderStatus: string
  graderType: string
  verdict: PassFailVerdict | null // null when the grader has not judged the trace
  reason: string | null
  failureDiagnostic: string | null
  confidence: VerdictConfidence | null
  source: GraderVerdictSource
  evaluatedAt: string | null
}

client.graders.saveVerdict(update: GraderVerdictUpdate): Promise<GraderVerdictOutcome>
client.graders.saveVerdictAll(updates: GraderVerdictUpdate[]): Promise<GraderVerdictOutcome[]>

type GraderVerdictUpdate = {
  graderId: string
  traceId: string
  verdict: PassFailVerdict
  reason?: string
  failureDiagnostic?: string
  confidence?: VerdictConfidence
}

type GraderVerdictOutcome = {
  graderId: string
  traceId: string
  verdict: PassFailVerdict
  action: "set"
}

​

experiments

client.experiments.list(params?: ListExperimentsParams): Promise<ExperimentPage>
client.experiments.get(experimentId: string): Promise<Experiment>
client.experiments.getRollup(experimentId: string): Promise<ExperimentRollupResult>
client.experiments.getRollupAll(experimentIds: string[]): Promise<ExperimentRollupResult[]>

type ListExperimentsParams = {
  datasetId?: string
  traceFunctionKey?: string
  gitBranch?: string
  experimentGroupId?: string
  status?: ExperimentStatus
  metadata?: Record<string, string>
  createdAfter?: Date | string
  createdBefore?: Date | string
  cursor?: string
  limit?: number // 1 to 100, default 20
}

type ExperimentStatus = "pending" | "completed" | "failed" | "interrupted"

type ExperimentPage = {
  experiments: Experiment[]
  nextCursor: string | null
  hasMore: boolean
}

type Experiment = {
  id: string
  name: string | null
  notes: string | null
  status: ExperimentStatus
  traceFunctionKey: string | null
  createdAt: string
  completedAt: string | null
  codeChangeDescription: string | null
  codeChangeTotals: { files: number; added: number; removed: number } | null
  experimentGroupId: string | null
  experimentGroupName: string | null
  datasetId: string | null
  datasetName: string | null
  datasetIds: string[]
  datasetNames: string[]
  attempts: number
  plannedTraceCount: number | null
  runBy: { id: string; fullName: string | null; email: string | null; imageUrl: string | null } | null
  githubEmail: string | null
  gitBranch: string | null
  commitSha: string | null
  baseSha: string | null
  experimentSha: string | null
  metadata: Record<string, string>
}

type ExperimentRollupResult = {
  experimentId: string
  totals: ExperimentTotals
  rollup: ExperimentRollup
}

// How each replay in the run ended. `total` counts one replay per trace per
// attempt, and `traces` counts distinct traces. `judging` counts traces, replays
// or originals, with an assertion verdict still being judged.
type ExperimentTotals = {
  total: number
  traces: number
  succeeded: number
  failed: number
  pending: number
  awaitingVerdicts: number
  errored: number
  withErrors: number
  ungradable: number
  judging: number // traces with an assertion verdict still being judged
  skipped: number
}

type ExperimentRollup = {
  traces: ExperimentTally // replayed traces
  skippedTraces: number
  verdicts: ExperimentTally // scored assertions and graders
  checks: ExperimentTally // raw checks before the pass threshold
  categories: { category: CategorySummary; verdicts: ExperimentTally }[]
  jitter: { disagreeing: number; attempts: number; traces: number }
}

// A trace or an assertion passes when at least 75 percent of its attempts
// passed. passing and failing count every verdict, compared to the original
// or not, so passing / (passing + failing) is the pass rate.
type ExperimentTally = {
  fixed: number
  stillPassing: number
  regressed: number
  stillFailing: number
  originalWithoutVerdict: number
  unpaired: number
  classified: number
  passing: number
  failing: number
  abstained: number // failing verdicts the judge abstained on in every attempt
}

​

Replay registry and command

export default defineReplayRegistry({
  checkout: { client: bitfab, fn: runCheckout },
  agent: {
    client: bitfab,
    fn: runAgent,
    traceFunctionKey: "support-agent",
    options: { mock: "marked", adaptInputs },
    optionsFactory: ({ params }) => ({
      mockOverride: createSupportAgentMock(params.scenario),
    }),
  },
})

​

seedTrace

seedTrace(
  traceFunctionKey: string,
  options: {
    input: unknown[]
    expected?: unknown
    fn?: (...args: any[]) => unknown
    metadata?: Record<string, unknown>
    sessionId?: string
    name?: string
    spanName?: string
    spanType?: SpanType
  },
): string

seedTrace<TArgs extends unknown[], TReturn>(
  traceFunctionKey: string,
  fn: (...args: TArgs) => TReturn,
  options?: {
    args?: TArgs
    metadata?: Record<string, unknown>
    sessionId?: string
    name?: string
  },
): Promise<string>

​

Seeding by running once

const bitfab = new Bitfab({ captureEnabled: false })

const traceId = await bitfab.seedTrace("agent-turn", runTicket, {
  args: ["T-1"],
  metadata: { caseUid: "c-1", suite: "smoke" },
  name: "T-1",
})

​

reseedTrace

reseedTrace(
  traceFunctionKey: string,
  fn: (...args: any[]) => unknown,
  options: { traceId: string },
): Promise<{ traceId: string; previousRunTraceId: string }>

​

reseedFromRegistry

reseedFromRegistry(
  registry: ReplayRegistry,
  pipeline: string,
  traceIds: readonly string[],
): Promise<{
  pipeline: string
  traceFunctionKey: string
  reseeded: { traceId: string; previousRunTraceId: string }[]
}>

​

seedFromRegistry

async function seedFromRegistry(
  registry: ReplayRegistry,
  pipeline: string,
  cases: readonly SeedCase[],
  options?: { run?: boolean },
): Promise<SeedResult>

​

registerMockOverride

registerMockOverride(override: MockOverride): void
registerMockOverride(resolver: MockOverrideResolver): void
registerMockOverride(match: NodeMatcher, value: MockValue): void
registerMockOverride(
  traceFunctionKey: string,
  override: MockOverride | MockOverrideResolver,
): void

​

clearMockOverrides

clearMockOverrides(): void

​

replay

async replay<TReturn>(
  traceFunctionKey: string,
  fn: (...args: any[]) => TReturn | Promise<TReturn>,
  options?: ReplayOptions,
): Promise<ReplayResult<TReturn>>

• A trace replays only when its root span has serializable inputs. It can also replay when it was instrumented through a framework handler. That handler’s recorded root input is serializable. If the original inputs were stubbed as non-serializable at capture time, the trace cannot be replayed
• fn may already be a withTrace- or withSpan-wrapped function. That function carries its trace function key and is used as-is. fn may instead be a plain callable. replay() wraps that callable under the key automatically. Either way, new spans link to the experiment via async context. Don’t wrap an already-wrapped function in a fresh closure. The closure has no trace function key, so replay() wraps the closure as the root, and the inner wrapped function records a second root, nesting a duplicate
• Inputs are deserialized from historical spans and passed positionally
• When codeChangeFiles is omitted, replay() auto-captures the code change. By default it captures a rename-aware diff of uncommitted working-tree changes (including untracked files) against HEAD. When the working tree is clean, it diffs against the branch yours was cut from instead (the parent branch in a stack, otherwise the point where you split from the default branch: origin/HEAD, else origin/main, origin/master, main, or master), so changes you already committed still show. The diff is bounded to 60 files, 500,000 bytes per file, and 2,000,000 bytes total. Set BITFAB_CODE_CHANGE_PATH to point at a JSON file with description and/or files instead, and that file wins over the git diff. BITFAB_CODE_CHANGE_BASE diffs against the merge-base with that ref instead (for example origin/main), which includes committed branch changes. BITFAB_DISABLE_CODE_CHANGE_CAPTURE opts the whole process out of auto-capture. Passing codeChangeFiles: null opts out one call instead
• When the Bitfab plugin sets BITFAB_REPLAY_RESULT_PATH, replay() writes the full ReplayResult, using serializeReplayResult, to that file after the run completes. A caller does not have to hand-parse stdout as a result. A write failure is logged. It never throws
• resume picks a stopped experiment back up by its experiment ID. The server returns the experiment’s saved trace selection, its attempt count, and which trace and attempt pairs already finished. Only the unfinished pairs run, and they record into the same experiment. A pair counts as unfinished when its trace never finished, including one Bitfab marked failed only because the stopped run went quiet. A pair that errored in the earlier run stays finished
• The experiment fixed its selection when it started. So resume throws a BitfabError when combined with traceIds, limit, datasetId, datasetIds, attempts, graderIds, onlyWithAssertions, or dryRun
• Database snapshots follow the experiment’s stored dbBranch setting, including its compute size and warm-up. A dbBranch passed with resume that disagrees is ignored, with a console warning
• The finished pairs come back first in items, with carriedOver: true. They carry originalTraceId, attempt, and the replay traceId. Their input, result, outputs, timings, and tokens are empty or null. Their error is set when the pair errored in the earlier run. onItemStart and onItemFinish do not fire for them, but the running totals count them
• A trace that was deleted after the experiment started is skipped, with a console warning naming it
• The server refuses a resume, with a 400 and a message saying why, when the experiment is already completed or failed, was started before resume was available, or was started as an experimental selective replay. It also refuses when the experiment was started, resumed, or received replay results in the last two minutes, since the run that started it may still be running. It also refuses when the code running now differs from the code the experiment started from, either in the working tree it ran or in its code change files. When the working tree differs, the message names the commit and tree the experiment started from (which includes any uncommitted changes it ran with), so you can return your working tree to that state your own way and resume again. force: true skips the last two refusals, never the others. An interrupted experiment is accepted right away, without the two-minute wait, and set back to pending for the resumed run
• Database branches the stopped run still held are released before the resume starts. A resume fails if Bitfab cannot release them. Try it again in that case
• Once the experiment is created, a replay that stops before it finishes marks the experiment interrupted through POST /api/sdk/replay/interrupt, waiting up to five seconds for an answer. A failed request (an error status, a timeout, or no connection) is never thrown. Only a pending experiment moves, so a completed one stays completed
• On SIGINT or SIGTERM in Node, replay() marks the experiment interrupted, then calls onInterrupt and leaves the rest to it. The run keeps going if the callback returns, so exit from it to stop. Without onInterrupt, it prints [replay] Experiment <id> interrupted. to stderr, or [replay] Experiment <id> stopped, but Bitfab could not mark it interrupted. It will be marked interrupted within 15 minutes. A resume in the next two minutes may be refused as still running. when the request failed, then re-sends the signal so the process ends as it normally would. When you installed your own handler for that signal, it is left to your handler instead. The listeners are removed just before the final request that completes and grades the experiment, so a signal arriving during it ends the process as it normally would and leaves the experiment to Bitfab’s own 15-minute check, and they are removed when the run ends
• When the run throws before it finishes, replay() marks the experiment interrupted and rethrows the same error. onInterrupt is not called
• A replay killed before it can mark itself, for example by SIGKILL, is marked interrupted by the server once it has gone 15 minutes with no replay activity. The server checks every five minutes. A running replay sends a heartbeat about once a minute, so only a process that stops sending it is marked interrupted
• An interrupted experiment is not graded. Resume it with resume, or mark it finished with Mark finished on the experiment page or save_experiment with status: "completed", which grades the traces that already finished

​

call

async call<T = unknown>(
  methodName: string,
  inputs?: Record<string, unknown>,
): Promise<T>

​

Framework Integrations

​

getLangGraphCallbackHandler

getLangGraphCallbackHandler(traceFunctionKey: string): BitfabLangGraphCallbackHandler

​

getLangGraphIntegration

getLangGraphIntegration(
  traceFunctionKey: string,
  options?: LangGraphIntegrationOptions,
): BitfabLangGraphIntegration

interface LangGraphIntegrationOptions {
  mockToolsOnReplay?: boolean | readonly string[] // default true; a string[] marks only those tool names
}

integration.middleware: BitfabAgentMiddleware
integration.instrument<TGraph>(graph: TGraph): TGraph
integration.createAgent<TAgent = BitfabLangGraphAgent>(params): Promise<TAgent>      // params of createAgent from "langchain"
integration.createDeepAgent<TAgent = BitfabLangGraphAgent>(params): Promise<TAgent>  // params of createDeepAgent from "deepagents"

interface BitfabLangGraphAgent {
  invoke(input: unknown, config?: Record<string, unknown>): Promise<any>
  stream(input: unknown, config?: Record<string, unknown>): Promise<AsyncIterable<any>>
  streamEvents(input: unknown, config: Record<string, unknown>, streamOptions?: Record<string, unknown>): AsyncIterable<any>
  batch(inputs: unknown[], config?: Record<string, unknown> | Record<string, unknown>[]): Promise<any[]>
  withConfig(config: Record<string, unknown>): BitfabLangGraphAgent
  getState(config: Record<string, unknown>): Promise<unknown>
}