Python SDK

# pip
pip install bitfab-py

# Poetry
poetry add bitfab-py

# uv
uv add bitfab-py

import os
from bitfab import Bitfab

bitfab = Bitfab(api_key=os.environ["BITFAB_API_KEY"])

Bitfab(api_key="...")

# Omit api_key entirely: the SDK reads BITFAB_API_KEY from the environment
Bitfab()

# Capture off: decorated functions still execute, but no spans are sent.
# Replay still records inside each item, and seed_trace records its one call.
Bitfab(api_key="...", capture_enabled=False)

# Leave it out and BITFAB_CAPTURE_ENABLED=0 or 1 decides
Bitfab(api_key="...")

# Pass a callable to defer resolution explicitly (resolved at first use):
Bitfab(api_key=lambda: os.environ.get("BITFAB_API_KEY"))

# Raises on the first traced call if no key resolves, instead of disabling quietly
Bitfab(api_key=os.environ.get("BITFAB_API_KEY"), strict=True)

order_service = bitfab.get_function("order-processing")

@order_service.span(mock_on_replay=True)
def load_order(order_id: str) -> Order:
    return db.orders.find_by_id(order_id)

@order_service.span(type="llm")
def classify_order(order: Order) -> Classification:
    return client.responses.parse(model=MODEL, input=build_prompt(order), text_format=Classification)

@order_service.span()
def validate_order(classification: Classification) -> dict:
    return {"valid": classification.confidence > 0.8}

# The root calls the decorated steps, so they record as its children
@order_service.span(type="agent")
def process_order(order_id: str) -> dict:
    order = load_order(order_id)
    classification = classify_order(order)
    return validate_order(classification)

process_order
├── load_order
├── classify_order
└── validate_order

# lib/bitfab_client.py -- single source of truth
import os
from bitfab import Bitfab
bitfab = Bitfab(api_key=os.environ["BITFAB_API_KEY"])
order_service = bitfab.get_function("order-processing")

# services/validate_order.py
from lib.bitfab_client import order_service

@order_service.span()
def validate_order(order_id: str) -> dict:
    return {"valid": True}

# services/process_order.py
from lib.bitfab_client import order_service
from services.validate_order import validate_order

@order_service.span()
def process_order(order_id: str) -> dict:
    validate_order(order_id)
    return {"order_id": order_id}

one_off_operation = bitfab.get_function("one-off-operation")

@one_off_operation.span()
def standalone_task() -> str:
    return "done"

outer_fn = bitfab.get_function("outer")
inner_fn = bitfab.get_function("inner")

@outer_fn.span(type="agent")
def outer():
    inner()  # Becomes a child of "outer"

@inner_fn.span(type="function")
def inner():
    pass

workflow_fn = bitfab.get_function("workflow")

@workflow_fn.span(type="function", capture_when="nested")
def helper(value: str) -> str:
    return transform(value)

@workflow_fn.span()
def workflow(value: str) -> str:
    return helper(value)

helper("standalone")  # Runs normally without creating a trace
workflow("nested")     # Captures helper as a child

ticket_triage = bitfab.get_function("ticket-triage")

@ticket_triage.trace(type="agent")
def triage(ticket: dict) -> dict:
    normalized = normalize_ticket(ticket)   # recorded, not decorated
    signals = extract_signals(normalized)   # recorded, not decorated
    return build_response(normalized, signals)

@bitfab.trace("ticket-triage", type="agent")
def triage(ticket: dict) -> dict:
    ...

triage
├─ normalize_ticket
│  ├─ redact_emails
│  └─ collapse_whitespace
├─ extract_signals
│  ├─ detect_keywords
│  └─ measure_length
└─ build_response
   └─ render_summary

ticket_triage = bitfab.get_function("ticket-triage")

@bitfab.node(name="Generate reply", type="llm")
def generate_reply(prompt: str) -> str:
    return call_model(prompt)

@bitfab.node(mock_on_replay=False)
def apply_new_policy(value: str) -> str:
    return value.strip()

@bitfab.node(capture=False)
def implementation_detail(value: str) -> str:
    return normalize(value)

@ticket_triage.trace(type="agent", mock_on_replay_default=True)
def triage(ticket: dict) -> dict:
    prompt = apply_new_policy(implementation_detail(ticket["body"]))
    return {"reply": generate_reply(prompt)}

import contextvars
from concurrent.futures import ThreadPoolExecutor

review_draft = bitfab.get_function("review-draft")

@bitfab.node(type="llm", name="review.check", variant=lambda draft, check: check)
def run_check(draft: str, check: str) -> dict:
    return call_model(draft, check)

@review_draft.trace(type="agent")
def review(draft: str) -> list[dict]:
    checks = ["tone", "facts", "safety"]
    with ThreadPoolExecutor(max_workers=len(checks)) as pool:
        submitted = [
            pool.submit(contextvars.copy_context().run, run_check, draft, check)
            for check in checks
        ]
        return [future.result() for future in submitted]

ticket_triage = bitfab.get_function("ticket-triage")

@ticket_triage.trace(
    max_captured_subtree_spans=100,  # cap discovered calls sent with inputs and outputs
    max_depth=10,                    # stop discovering below this call depth
    exclude=["parse_row"],           # never record these function names
    include_wrappers=True,           # record decorator wrappers too (off by default)
)
def triage(ticket: dict) -> dict:
    ...

ticket_detail = bitfab.get_function("ticket-detail")
ticket_workflow = bitfab.get_function("ticket-workflow")

@ticket_detail.trace()
def build_ticket_detail(ticket: dict) -> dict:
    return enrich_ticket(ticket)

@ticket_workflow.trace()
def process_ticket(ticket: dict) -> dict:
    return build_ticket_detail(ticket)

process_ticket(ticket)
# Creates two independent traces:
# ticket-workflow -> build_ticket_detail -> enrich_ticket
# ticket-detail   -> enrich_ticket

ticket_detail = bitfab.get_function("ticket-detail")
ticket_workflow = bitfab.get_function("ticket-workflow")

@ticket_detail.span(type="function")
def build_ticket_detail(ticket: dict) -> dict:
    return enrich_ticket(ticket)

@ticket_workflow.trace(type="agent")
def process_ticket(ticket: dict) -> dict:
    return build_ticket_detail(ticket)

process_ticket(ticket)
# MixedTracingError: Opt-in and opt-out tracing can't be mixed: @span (opt-in)
# was entered inside a @trace call (opt-out). Inside a @trace subtree,
# configure a function with @node instead, or trace this workflow with @span only.

bitfab = Bitfab(trace_across_threads=True)

SpanType = Literal[
    "llm",        # LLM calls
    "agent",      # Agent workflows
    "function",   # Function calls
    "guardrail",  # Safety checks
    "handoff",    # Human handoffs
    "custom"      # Default
]

order_processing = bitfab.get_function("order-processing")
safety_check = bitfab.get_function("safety-check")

# Function name is automatically captured as span name
@order_processing.span()
def process_order(order_id: str) -> dict:
    return {"order_id": order_id}
# Span name: "process_order"

# Override with name option
@order_processing.span(name="OrderProcessor")
def process_order(order_id: str) -> dict:
    return {"order_id": order_id}
# Span name: "OrderProcessor"

# Set span type
@safety_check.span(type="guardrail")
def check_content(content: str) -> dict:
    return {"safe": True}

from bitfab import finalizers

chat_fn = bitfab.get_function("chat")

@chat_fn.span(type="llm", finalize=finalizers.openai_chunks)
async def chat(messages):
    stream = await client.chat.completions.create(
        model="gpt-4o", messages=messages, stream=True
    )
    async for chunk in stream:
        yield chunk  # caller still receives every chunk

# The span records { text, finish_reason, usage, tool_calls } in the background.

chat_fn = bitfab.get_function("chat")

@chat_fn.span(type="llm", finalize=lambda chunks: {"text": "".join(
    c.choices[0].delta.content or "" for c in chunks
)})
async def chat(messages):
    ...
    async for chunk in stream:
        yield chunk

from bitfab import get_current_span

order_processing = bitfab.get_function("order-processing")

@order_processing.span(type="function")
def process_order(order_id: str) -> dict:
    user_id = get_current_user()
    get_current_span().add_context({"user_id": user_id, "order_id": order_id})
    return {"order_id": order_id, "status": "completed"}

get_current_span().add_context({"user_id": "u-123"})
get_current_span().add_context({"request_id": "req-789"})
# Result: contexts: [{"user_id": "u-123"}, {"request_id": "req-789"}]

from bitfab import get_current_span

classification_fn = bitfab.get_function("classification")

@classification_fn.span(type="llm")
def classify_text(text: str) -> str:
    prompt = f"Classify the following text: {text}"
    get_current_span().set_prompt(prompt)
    result = llm.complete(prompt)
    return result

from bitfab import get_current_trace

order_processing = bitfab.get_function("order-processing")

@order_processing.span(type="function")
def process_order(order_id: str) -> dict:
    trace = get_current_trace()

    # Set session ID (stored as database column, filterable in dashboard)
    trace.set_session_id("session-123")

    # Name the trace (its title in Bitfab, searchable and filterable)
    trace.set_name(f"Order {order_id}")

    # Set trace metadata (indexed for exact-match search)
    trace.set_metadata({"region": "us-west-2", "environment": "production"})

    # Add context entries (stored as key-value pairs, accumulates across calls)
    trace.add_context({"workflow": "checkout-flow", "batch_id": "batch-2024-01"})

    return {"order_id": order_id, "status": "completed"}

page = client.traces.search(
    caller_metadata={"ticket_id": "ABC-123", "tenant": "tanagram"},
    trace_function_key="order-processing",
    limit=50,
)

exact = client.traces.search(name="Order ABC-123")
partial = client.traces.search(name_contains="order", limit=50)

for experiment_id in (baseline_experiment_id, candidate_experiment_id):
    traces = client.traces.search(experiment_id=experiment_id, limit=500)["traces"]
    total_ms = sum(t["durationMs"] or 0 for t in traces)
    total_tokens = sum((t["tokens"] or {}).get("total") or 0 for t in traces)
    print(experiment_id, len(traces), total_ms, total_tokens)

from bitfab import get_current_trace

order_processing = bitfab.get_function("order-processing")

@order_processing.span(type="function")
def process_order(order_id: str) -> dict:
    if is_health_check(order_id):
        get_current_trace().drop()

    return {"order_id": order_id, "status": "completed"}

trace = client.get_trace(trace_id)
trace.add_context({"refund_status": "approved"})
trace.set_metadata({"region": "us-west"})
trace.set_session_id("session_xyz")
trace.set_name("Ticket 4521")

span = client.get_trace_span(trace_id, name="GenerateAnswer")
first = client.get_trace_span(
    trace_id, name="GenerateAnswer", occurrence="first"
)
first_model_call = client.get_trace_span(trace_id, type="llm", occurrence="first")
exact = client.get_trace_span(trace_id, id=span_id)

page = client.spans.list(trace_id, fields=["input"])
while True:
    for listed in page["spans"]:
        print(listed["type"], listed["name"], listed["input"])
    if not page["hasMore"]:
        break
    page = client.spans.list(trace_id, fields=["input"], cursor=page["nextCursor"])

span = client.get_span(span_id)

risky_service = bitfab.get_function("risky-service")

@risky_service.span()
def risky():
    raise ValueError("error")

try:
    risky()
except ValueError:
    pass
# Span records error and timing

from bitfab import flush_traces

if not flush_traces(timeout=30.0):
    raise RuntimeError("Bitfab traces were not delivered before the deadline")

my_function_key = bitfab.get_function("my-function-key")

@my_function_key.span()
def my_function(text: str) -> dict:
    return {"processed": text.upper()}

result = bitfab.replay(my_function, limit=5)

# Or replay specific traces by ID
result = bitfab.replay(my_function, trace_ids=["trace-abc", "trace-def"])

print(f"Experiment: {result['experiment_url']}")
for item in result["items"]:
    print(f"  Input: {item['input']}")
    print(f"  Result: {item['result']}")
    print(f"  Original: {item['original_output']}")
    print(f"  Duration (ms): {item['original_duration_ms']}")
    print(f"  Tokens: {item['tokens']}")       # {"input", "output", "cached", "total"} or None
    print(f"  Model: {item['model']}")

{
    "items": [
        {
            "input": [...],             # The inputs passed to fn
            "result": ...,              # What fn returned
            "original_output": ...,     # What the original trace produced
            "error": None | str,        # Error message if fn raised
            "duration_ms": int | None,  # How long THIS replay took, in ms
            "original_duration_ms": int | None,  # The original trace's duration
            "original_tokens": {...} | None,     # Original trace token usage
            "original_model": str | None,        # Original model name
            "tokens": {                 # Replayed run token usage, or None
                "input": int | None,
                "output": int | None,
                "cached": int | None,
                "total": int | None,
            } | None,
            "model": str | None,        # Deprecated alias for original_model
            "db_branch_timings": dict | None,  # Per-phase branch provisioning timings
            "trace_id": str | None,     # Server trace ID for the replayed execution
            "db_snapshot_ref": dict | None,  # The source trace's snapshot pin, if any
            "carried_over": bool,       # True when a resume kept this item from the earlier run
        }
    ],
    "experiment_id": "...",
    "experiment_url": "...",
    "test_run_id": "...",   # Deprecated alias for experiment_id
    "test_run_url": "..."   # Deprecated alias for experiment_url
}

# scripts/replay_registry.py
from bitfab import ReplayRegistry
from my_app.agent import graph
from my_app.bitfab_client import bitfab

my_agent = bitfab.get_function("my-agent")

handler = my_agent.get_langgraph_callback_handler()


def replay_my_agent(state):
    config = {"callbacks": [handler], "configurable": build_replay_config()}
    return graph.invoke(state, config=config)


registry = ReplayRegistry()
registry.register(
    "agent", bitfab, replay_my_agent, trace_function_key="my-agent"
)

article_pipeline = bitfab.get_function("process-article")


@article_pipeline.span(name="fetch-article-from-db", mock_on_replay=True)
def fetch_article_from_db(article_id: str) -> Article:
    return db.articles.find_by_id(article_id)


@article_pipeline.span(name="summarize-article")
def summarize_article(article: Article) -> Summary:
    # Real summarization, no flag -- this is what we're iterating on.
    return Summary(...)


@article_pipeline.span(name="process-article")
def process_article(article_id: str) -> Summary:
    return summarize_article(fetch_article_from_db(article_id))


# During replay, fetch-article-from-db returns its recorded output;
# summarize-article runs real so you can iterate on it.
result = bitfab.replay(process_article, limit=10)

from bitfab import MockOverride

result = bitfab.replay(
    process_article,
    mock="none",  # run everything real...
    mock_override=MockOverride(
        # ...except this span, which gets the value you supply
        match=lambda node: node.span_name == "fetch-article-from-db",
        value={"id": "fixed", "title": "Fixed title"},  # flat value
    ),
)

value=lambda ctx: {**ctx.get_original_output(), "score": 1}

bitfab.register_mock_override(
    MockOverride(match=lambda node: node.type == "llm", value={"label": "refund"})
)
# Ordered form (equivalent): bitfab.register_mock_override(match, value)
bitfab.clear_mock_overrides()

client.register_mock_override(
    "classify-intent",
    lambda ctx: {"label": str(ctx.inputs[0])},
)

client.register_mock_override(
    "shared-workflow",
    MockOverride(
        match=lambda node: node.span_name == "Classifier",
        value={"label": "refund"},
    ),
)

from bitfab import NO_MOCK_OVERRIDE

client.register_mock_override(
    lambda ctx: (
        {"label": str(ctx.inputs[0])}
        if ctx.node.trace_function_key == "classify-intent"
        else NO_MOCK_OVERRIDE
    )
)

# Recorded as (user_id, limit); current signature is (opts: dict).
def adapt(args, kwargs, ctx):
    user_id, limit = args
    return [{"user_id": user_id, "limit": limit}], {}


result = bitfab.replay(my_function, adapt_inputs=adapt)

# scripts/replay_adapters/extraction.py
def adapt_inputs(args, kwargs, ctx):
    user_id, limit = args
    return [{"user_id": user_id, "limit": limit}], {}

# scripts/replay_registry.py
from bitfab import ReplayRegistry
from my_app.bitfab_client import bitfab
from my_app.workflow import my_function
from replay_adapters.extraction import adapt_inputs

registry = ReplayRegistry()
registry.register(
    "workflow", bitfab, my_function, adapt_inputs=adapt_inputs
)

with open("src/foo.py") as f:
    before = f.read()

# ...edit src/foo.py...

with open("src/foo.py") as f:
    after = f.read()

result = bitfab.replay(
    my_function,
    code_change_description="fix off-by-one in retry logic",
    code_change_files=[{"path": "src/foo.py", "before": before, "after": after}],
)

result = bitfab.replay(my_function, resume="3f0c9c1e-5d2a-4b7e-9a61-2c8f0e4d7b13")

for item in result["items"]:
    if item["carried_over"]:
        continue  # finished in the earlier run
    print(item["result"])

def report(interrupt):
    print(f"Experiment {interrupt['experiment_id']} interrupted by {interrupt['signal']}")

bitfab.replay(my_function, on_interrupt=report)

result = bitfab.replay(my_function, limit=limit)

# Human-readable summary goes to stderr, so stdout stays pure JSON.
print(f"Experiment: {result['experiment_url']}", file=sys.stderr)
print(f"Items:      {len(result['items'])}", file=sys.stderr)

# Then: full structured dump, ready for json.loads. The SDK serializer retains
# useful fields from trace_error and replay_error exception objects.
print(serialize_replay_result(result))

from dotenv import load_dotenv
from bitfab import ReplayRegistry
from lib.bitfab_client import bitfab
from scripts.replay_mocks import create_search_mock
from services.extraction import extract_memories
from services.search import search_documents

load_dotenv()

registry = ReplayRegistry()
registry.register("extraction", bitfab, extract_memories)
registry.register(
    "search",
    bitfab,
    search_documents,
    # Plain handler functions need the key explicitly. A function decorated
    # with @trace or @span carries its key, so trace_function_key can be omitted.
    trace_function_key="my-search-pipeline",
    mock="marked",
    options_factory=lambda ctx: {
        "mock_override": create_search_mock(ctx.params.get("scenario"))
    },
)

def grade(progress):
    item = progress["item"]
    target = dict(
        original_trace_id=item["original_trace_id"],
        attempt=item["attempt"],
        experiment_id=progress["experiment_id"],
    )
    assertions = client.traces.get_assertions(item["original_trace_id"])
    for assertion in assertions["assertions"]:
        if assertion["approvalState"] != "approved":
            continue
        if item["error"] is not None:
            client.verdicts.skip(
                annotation=f"replay errored: {item['error']}",
                assertion_id=assertion["id"],
                **target,
            )
            continue
        assessment = {
            key: assertion[key]
            for key in (
                "assertion",
                "passCriteria",
                "failCriteria",
                "targetOnEvaluatedTrace",
            )
        }
        judgement = judge(assessment, item["result"])
        client.verdicts.save(
            verdict="pass" if judgement.passed else "fail",
            annotation=judgement.summary,
            assertion_id=assertion["id"],
            **target,
        )


registry = ReplayRegistry().register(
    "support-agent", client, run_agent, on_item_finish=grade
)

def grade_where_it_ran(event):
    item = event["item"]
    assertions = client.traces.get_assertions(item["original_trace_id"])
    for assertion in assertions["assertions"]:
        if assertion["approvalState"] != "approved":
            continue
        judgement = judge(assertion, collected_assertions(), item["result"])
        client.verdicts.save(
            verdict="pass" if judgement.passed else "fail",
            annotation=judgement.summary,
            trace_id=item["trace_id"],
            assertion_id=assertion["id"],
        )


registry = ReplayRegistry().register(
    "support-agent",
    client,
    run_agent,
    concurrency=ReplayConcurrency(
        primitive="process", on_item_finish_in_child_process=grade_where_it_ran
    ),
)

def stream_progress(progress):
    dashboard.update(done=progress["completed"], total=progress["total"])


registry = ReplayRegistry().register(
    "support-agent",
    client,
    run_agent,
    on_item_finish=stream_progress,
    concurrency=ReplayConcurrency(
        primitive="process", on_item_finish_in_child_process=grade_where_it_ran
    ),
)

# services/agent/agent/support_agent.py
support_agent_fn = bitfab.get_function("support-agent")


def create_support_agent(config: AgentConfig) -> SupportAgent:
    ticket_gateway = TicketGatewayClient(config.ticket_gateway_url)

    @support_agent_fn.span(name="lookup-ticket", type="tool", mock_on_replay=True)
    async def lookup_ticket(ticket_id: str) -> dict:
        return await ticket_gateway.lookup(ticket_id)

    @support_agent_fn.span(name="answer-question", type="agent")
    async def answer_question(question: str) -> dict:
        ticket = await lookup_ticket(parse_ticket_id(question))
        return {"reply": f"Ticket {ticket['id']} is {ticket['status']}."}

    return SupportAgent(answer_question=answer_question)

# services/agent/scripts/replay_registry.py
from dotenv import load_dotenv
from bitfab import ReplayRegistry
from agent.bitfab_client import bitfab
from agent.support_agent import create_support_agent, load_agent_config

load_dotenv()

support_agent = create_support_agent(load_agent_config())

registry = ReplayRegistry()
registry.register("support-agent", bitfab, support_agent.answer_question, mock="marked")

cd services/agent
PYTHONPATH=. bitfab-replay --registry scripts/replay_registry.py support-agent --dry-run
PYTHONPATH=. bitfab-replay --registry scripts/replay_registry.py support-agent --limit 1

# services/app/scripts/replay_registry.py
from agent_core import bitfab
from app.assistant import build_agent_tools, create_agent_runner
from bitfab import ReplayRegistry

runner = create_agent_runner()


def rebuild_tools(args, kwargs, ctx):
    prompt, recorded_options = args
    return [prompt, {**recorded_options, "tools": build_agent_tools()}], kwargs


registry = ReplayRegistry()
registry.register("agent-run", bitfab, runner.run, mock="marked", adapt_inputs=rebuild_tools)

from bitfab import Bitfab
from services.agent import run_ticket

bitfab = Bitfab(api_key="...", capture_enabled=False)

trace_id = bitfab.seed_trace(
    "agent-turn",
    run_ticket,
    kwargs={"ticket_id": "T-1"},
    metadata={"case_uid": "c-1", "suite": "smoke"},
    name="T-1",
)

bitfab.replay(run_ticket, trace_ids=[trace_id])

result = bitfab.reseed_trace("agent-turn", run_ticket, trace_id="3f2a...")
result["trace_id"], result["previous_run_trace_id"]

bitfab-seed --registry scripts/replay_registry.py extraction --from-trace 3f2a...

bitfab-replay --registry scripts/replay_registry.py extraction --seed cases.jsonl

Bitfab(
    api_key: str,                    # Required
    service_url: str | None = None,  # Default: https://bitfab.ai
    env_vars: dict[str, str] | None = None,  # For local function execution
    capture_enabled: bool | None = None,  # Capture traced calls; None reads BITFAB_CAPTURE_ENABLED
    simulation_plan: bool = True,    # Read and apply the sim plan
    baml_client: Any = None          # Generated BAML client (for wrap_baml)
)

saved = bitfab.datasets.save(
    "checkout-agent",
    "Refund failures",
    description="Checkout runs where the refund was declined",
)
dataset_id = saved["dataset"]["id"]

added = bitfab.datasets.add_traces(dataset_id, [trace_id])
if added["skippedTraceIds"]:
    print("not in this trace function:", added["skippedTraceIds"])

trace_ids = bitfab.datasets.list_traces(dataset_id)["traceIds"]
bitfab.datasets.add_graders(dataset_id, [grader_id])
run = bitfab.datasets.rerun_graders(dataset_id)["run"]
print(run["status"], run["result"])

result = client.traces.get_assertions(item.original_trace_id)

client.verdicts.save_all(
    [
        {
            "originalTraceId": item.original_trace_id,
            "attempt": item.attempt,
            "assertionId": assertion["id"],
            "verdict": "pass" if judge(assertion, output) else "fail",
            "annotation": explain(assertion, output),
        }
        for assertion in result["assertions"]
        if assertion["approvalState"] == "approved"
    ],
    experiment_id=experiment_id,
)

verdicts = client.verdicts.get(item.trace_id)
for verdict in verdicts["assertions"]:
    print(verdict["assertion"], verdict["verdict"], verdict["annotation"])

graded = client.graders.get_verdicts(trace_ids=[item.trace_id])

page = bitfab.experiments.list(
    dataset_id=dataset_id,
    git_branch="dev",
    metadata={"schedule": "eod"},
    status="completed",
    limit=30,
)
rollups = bitfab.experiments.get_rollup_all([e["id"] for e in page["experiments"]])
for result in rollups:
    for row in result["rollup"]["categories"]:
        verdicts = row["verdicts"]
        print(row["category"]["title"], f"{verdicts['passing']}/{verdicts['passing'] + verdicts['failing']}")