[ { "repeat": 1, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 4, "started_at": "2026-09-20T16:13:57.770Z", "latency_ms": 9736.763125000001, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.82, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "c87a1774-f33d-4972-ac5d-6150364135d3", "answer_receipt_ids": [ "gen-1789920846-32aSaUYRZyczMSUBFntf" ], "price_microusd": 106, "charged_cents": 0, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 1, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 4, "started_at": "2026-09-20T16:14:07.508Z", "latency_ms": 6592.854292, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920851-1viTgPvRE78oocgg6rIM" ], "price_microusd": 3796, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 9, "started_at": "2026-09-20T16:14:16.748Z", "latency_ms": 13585.229041000002, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.54, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "0923cd8e-b436-4b8b-8cf6-d44f0182e950", "answer_receipt_ids": [ "gen-1789920868-RD7WV0K8vcmLddP9xy4d" ], "price_microusd": 2145, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 9, "started_at": "2026-09-20T16:14:30.334Z", "latency_ms": 5538.5453750000015, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920873-RpjQBLZkVZzbWXBCwN9z" ], "price_microusd": 2344, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 15, "started_at": "2026-09-20T16:14:38.269Z", "latency_ms": 11338.533583000004, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.49, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "655e099e-9fd0-4b23-ae3a-c314e6c37b02", "answer_receipt_ids": [ "gen-1789920885-106k4Wy84pAn8o1MUytN" ], "price_microusd": 4152, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 15, "started_at": "2026-09-20T16:14:49.608Z", "latency_ms": 9416.724624999995, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920895-VcP69x9O1rhZIevPEwQM" ], "price_microusd": 4116, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 19, "started_at": "2026-09-20T16:14:59.025Z", "latency_ms": 14519.380458, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.68, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "71855bb3-df35-4d2c-9f6f-16f26b558fd2", "answer_receipt_ids": [ "gen-1789920910-EpcaWDUMl0P1XxHhUqDu" ], "price_microusd": 5671, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 16, "started_at": "2026-09-20T16:15:04.029Z", "latency_ms": 11983.670125000004, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920913-cszYqG3oYE4SZK7AEBTy" ], "price_microusd": 2977, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 19, "started_at": "2026-09-20T16:15:13.545Z", "latency_ms": 11226.0055, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920921-QfTcra0Zv6sgDO1e0H1G" ], "price_microusd": 5918, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 21, "started_at": "2026-09-20T16:15:29.404Z", "latency_ms": 8413.095291999998, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 1, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "21aa674a-a8fd-4d0e-90b4-d14684de6bc8", "answer_receipt_ids": [ "gen-1789920937-2y9JQXe9el4YoGUN8bpO" ], "price_microusd": 51, "charged_cents": 0, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 1, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 23, "started_at": "2026-09-20T16:15:36.054Z", "latency_ms": 6858.542083000008, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920940-y9uvDJVzL7TwqOknmIx0" ], "price_microusd": 3518, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 24, "started_at": "2026-09-20T16:15:37.052Z", "latency_ms": 8756.409499999994, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920943-TyMdMOrL4veDegRJKQ1y" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 25, "started_at": "2026-09-20T16:15:37.817Z", "latency_ms": 15598.847917000006, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.47, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "5a45c3ed-f7b0-41dd-a318-2ba881753ca7", "answer_receipt_ids": [ "gen-1789920949-oJxtcZ91D05hXg0rv81B" ], "price_microusd": 6074, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 24, "started_at": "2026-09-20T16:15:45.809Z", "latency_ms": 14126.177291999993, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.93, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "2a99b9d8-911d-4d47-82ce-aaa54bb04f9a", "answer_receipt_ids": [ "gen-1789920959-Ztcu8thDATLsL4D1K8Ju" ], "price_microusd": 96, "charged_cents": 0, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 1, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 25, "started_at": "2026-09-20T16:15:53.417Z", "latency_ms": 13072.538750000007, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920962-hBRlHGw3VlnWD2hcAVFE" ], "price_microusd": 6041, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 30, "started_at": "2026-09-20T16:16:09.766Z", "latency_ms": 7663.948499999999, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789920974-NAuvsl9nP29ykgVHfa30" ], "price_microusd": 3652, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 30, "started_at": "2026-09-20T16:16:17.430Z", "latency_ms": 12412.298500000004, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.8, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "ba084737-7917-4463-9397-2e571cc7f618", "answer_receipt_ids": [ "gen-1789920989-HEkjGqfAP8BFH1oPhP41" ], "price_microusd": 86, "charged_cents": 0, "answer": { "records_per_group": 56 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": 56 } ] } }, { "repeat": 1, "prompt_id": "high-stakes-06", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 33, "started_at": "2026-09-20T16:16:27.757Z", "latency_ms": 6848.032124999998, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ null ], "price_microusd": 0, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 41, "started_at": "2026-09-20T16:17:04.480Z", "latency_ms": 7109.833916000003, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921030-2fOmc5ehDmirXz1QRSKJ" ], "price_microusd": 891, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 1, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 42, "started_at": "2026-09-20T16:17:06.241Z", "latency_ms": 13541.285083000024, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.78, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "0fc9a91c-3b55-420c-b3ea-b91d6d432f45", "answer_receipt_ids": [ "gen-1789921037-E4AQdni8bFJrxAXlcai2" ], "price_microusd": 2584, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 41, "started_at": "2026-09-20T16:17:11.589Z", "latency_ms": 12131.679457999999, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes" ], "confidence": 0.84, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "5b9a0cc3-3c18-4caf-97aa-4c62fde7ba2a", "answer_receipt_ids": [ "gen-1789921042-KLEM5lCMqfvdgaZGYijM" ], "price_microusd": 925, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 1, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 42, "started_at": "2026-09-20T16:17:19.782Z", "latency_ms": 6247.611540999991, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921043-BHa7TXLNuzR8KFMF8B6o" ], "price_microusd": 2550, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 1, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 45, "started_at": "2026-09-20T16:17:23.721Z", "latency_ms": 8708.323082999996, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921049-UzurIRJLhpfmAEZxoT1n" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 4, "started_at": "2026-09-20T16:18:03.742Z", "latency_ms": 9495.90849999999, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.83, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "9bacd833-7994-49c1-9aa5-007f3785c77b", "answer_receipt_ids": [ "gen-1789921092-t37s7OmvKbb50CNv4hg8" ], "price_microusd": 81, "charged_cents": 0, "answer": { "seconds": 20 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": 20 } ] } }, { "repeat": 2, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 4, "started_at": "2026-09-20T16:18:13.238Z", "latency_ms": 5191.205083000008, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921096-k94gMAWnIrsqYCWkzX0z" ], "price_microusd": 2550, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 13, "started_at": "2026-09-20T16:18:34.694Z", "latency_ms": 9289.363249999995, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921121-f90ZDqcUHqXFTx1KdgOE" ], "price_microusd": 2344, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 14, "started_at": "2026-09-20T16:18:36.468Z", "latency_ms": 11380.866625000024, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921124-DHMNaZ6TQNhFVDqFXWSW" ], "price_microusd": 5635, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 13, "started_at": "2026-09-20T16:18:43.984Z", "latency_ms": 15459.498916000011, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.61, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "854a7a3f-e9a7-4446-a75d-49051ac5500b", "answer_receipt_ids": [ "gen-1789921136-HCnDCSkQuJnIfwq83Nqm" ], "price_microusd": 2145, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 14, "started_at": "2026-09-20T16:18:47.849Z", "latency_ms": 15147.289667000005, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.72, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "5c6d7a2b-a8d5-47bc-9041-6de4a5ec1203", "answer_receipt_ids": [ "gen-1789921139-56uOcmEUxAIgh1opEgdn" ], "price_microusd": 5954, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 16, "started_at": "2026-09-20T16:18:56.529Z", "latency_ms": 9757.590124999988, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921145-RAzUKzGUG6L4nivcV7dq" ], "price_microusd": 891, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 2, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 16, "started_at": "2026-09-20T16:19:06.286Z", "latency_ms": 9721.519582999987, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes" ], "confidence": 0.85, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "ee4525ab-302c-4110-955e-6ba0ef0ea41a", "answer_receipt_ids": [ "gen-1789921154-E6T32zMswFkqaW3QxsXe" ], "price_microusd": 925, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 2, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 24, "started_at": "2026-09-20T16:19:32.393Z", "latency_ms": 7619.620750000002, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921176-m1qhRyuXSJd4YNBDZCkt" ], "price_microusd": 6041, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 26, "started_at": "2026-09-20T16:19:34.814Z", "latency_ms": 8882.794166000036, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921181-YEKLDBdTWvIIlHEnfpjV" ], "price_microusd": 3796, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 24, "started_at": "2026-09-20T16:19:40.013Z", "latency_ms": 13460.186000000045, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.37, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "ed05bdc6-8327-4598-afe0-65961fde7045", "answer_receipt_ids": [ "gen-1789921189-pcutvrKzCywrzUDPdToD" ], "price_microusd": 6074, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 26, "started_at": "2026-09-20T16:19:43.698Z", "latency_ms": 13113.887790999957, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.85, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "233981ec-841b-48e4-867b-d09918f1b6f5", "answer_receipt_ids": [ "gen-1789921196-mKUOLBbH6NdTCeEtmyCD" ], "price_microusd": 106, "charged_cents": 0, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 2, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 30, "started_at": "2026-09-20T16:19:56.812Z", "latency_ms": 7196.894042, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921200-sMBNe1ZjehV3LrIcsoF2" ], "price_microusd": 4116, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 30, "started_at": "2026-09-20T16:20:04.010Z", "latency_ms": 14705.216749999963, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.5, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "4b89c2be-ae66-4912-bc96-6cb611a46f0c", "answer_receipt_ids": [ "gen-1789921215-IQ5opdeQcMuge9pi212M" ], "price_microusd": 4152, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 32, "started_at": "2026-09-20T16:20:13.251Z", "latency_ms": 9976.377916999976, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921220-d5pGwaleFPqflM6Pu5S2" ], "price_microusd": 3652, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 35, "started_at": "2026-09-20T16:20:20.575Z", "latency_ms": 6700.311625000031, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921224-5Xw30kKpF3bPo9Tc7XMv" ], "price_microusd": 3518, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 32, "started_at": "2026-09-20T16:20:23.227Z", "latency_ms": 10881.372415999998, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.81, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "47ffa13a-2950-41b1-8db6-05131014fd9c", "answer_receipt_ids": [ "gen-1789921233-knAx64XEFp6n8AQ99b29" ], "price_microusd": 86, "charged_cents": 0, "answer": { "records_per_group": 56 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": 56 } ] } }, { "repeat": 2, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 36, "started_at": "2026-09-20T16:20:34.109Z", "latency_ms": 11418.792541000003, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 0.99, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "e74dc57a-7d9f-4ab8-b02f-69ee9fc1514e", "answer_receipt_ids": [ "gen-1789921245-gjkfBP62ipp2NiQbeHv5" ], "price_microusd": 51, "charged_cents": 0, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 2, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 38, "started_at": "2026-09-20T16:20:46.835Z", "latency_ms": 9940.108542000002, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921254-hL915nixRrgWfREnQu9x" ], "price_microusd": 2977, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 43, "started_at": "2026-09-20T16:21:08.460Z", "latency_ms": 6331.512792000023, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921271-1H0fjPEJAlcWXBFfK33E" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 2, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 47, "started_at": "2026-09-20T16:21:14.792Z", "latency_ms": 11163.057583999995, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.92, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "0699792d-0954-4b6b-a946-fd1399fbf413", "answer_receipt_ids": [ "gen-1789921284-yd3lJH4bps9SXzXMfe0Y" ], "price_microusd": 96, "charged_cents": 0, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 2, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 47, "started_at": "2026-09-20T16:21:25.956Z", "latency_ms": 11379.240417000023, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921294-2GEWCghZO0awxrws68x3" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 3, "started_at": "2026-09-20T16:21:43.924Z", "latency_ms": 6556.170582999999, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921306-LTwHBu7anbofL19CU3AD" ], "price_microusd": 6041, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-08", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 3, "started_at": "2026-09-20T16:21:50.481Z", "latency_ms": 15488.636457999994, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.5, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "e299c13f-28c9-4f3c-98b8-b6fa56272ab9", "answer_receipt_ids": [ "gen-1789921322-c4S5xNAhrTSUyvvRE9MU" ], "price_microusd": 6074, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 26, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 9, "started_at": "2026-09-20T16:22:19.580Z", "latency_ms": 7892.241625000024, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921345-zNZhUDpBCvFPTtuTsALN" ], "price_microusd": 2550, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 11, "started_at": "2026-09-20T16:22:22.101Z", "latency_ms": 11339.725916999974, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.82, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "8fd2a77e-55f8-4496-aef2-067225cfdc2c", "answer_receipt_ids": [ "gen-1789921352-uVrqbyUJiS2sVrrauMEx" ], "price_microusd": 106, "charged_cents": 0, "answer": { "plan": "B", "cost_dollars": 7.8 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": true, "actual": "B" }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": 7.8 } ] } }, { "repeat": 3, "prompt_id": "lookup-06", "bucket": "lookup", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 8, "started_at": "2026-09-20T16:22:25.199Z", "latency_ms": 10290.20633399993, "decision": { "lane": "fast", "recommended": "fast", "reason": [], "confidence": 0.99, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "fast", "final_lane": "fast", "fallback": false, "router_transaction_id": "5e44dd99-7d4e-442f-86b4-4da717552cb1", "answer_receipt_ids": [ "gen-1789921354-LRLbRR6HAslsCrmESEC0" ], "price_microusd": 51, "charged_cents": 0, "answer": { "day": "Thursday", "time_utc": "09:30 UTC" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "day", "value": "Thursday", "pass": true, "actual": "Thursday" }, { "op": "eq", "field": "time_utc", "value": "09:30", "pass": false, "actual": "09:30 UTC" } ] } }, { "repeat": 3, "prompt_id": "multi-step-10", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 9, "started_at": "2026-09-20T16:22:27.473Z", "latency_ms": 10373.796625000017, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.79, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "4181963f-3c96-4bd2-b768-81c3fd2d6045", "answer_receipt_ids": [ "gen-1789921355-KyR599Sr4NXgy6cFd1QR" ], "price_microusd": 2584, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "seconds", "value": 25, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-09", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 11, "started_at": "2026-09-20T16:22:33.441Z", "latency_ms": 8913.00524999993, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921359-Ck0AL6dH2eKvIcXFluj8" ], "price_microusd": 3796, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "plan", "value": "B", "pass": false, "actual": null }, { "op": "eq", "field": "cost_dollars", "value": 6.8, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 12, "started_at": "2026-09-20T16:22:35.490Z", "latency_ms": 9682.218166999985, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921362-quWa535YgwSONnlY8KKw" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-02", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 12, "started_at": "2026-09-20T16:22:45.172Z", "latency_ms": 9767.08683299995, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.94, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "604151dd-0ab4-474c-bc3b-deb7215714d6", "answer_receipt_ids": [ "gen-1789921374-veVL8QurG4uPv4o10yC0" ], "price_microusd": 96, "charged_cents": 0, "answer": { "total_dollars": 66.6 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "total_dollars", "value": 67.2, "pass": false, "actual": 66.6 } ] } }, { "repeat": 3, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 19, "started_at": "2026-09-20T16:22:59.009Z", "latency_ms": 7484.37200000009, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921383-R8NrcJSvceVZO0ZcbtuD" ], "price_microusd": 5918, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-06", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 19, "started_at": "2026-09-20T16:23:06.493Z", "latency_ms": 14306.81320799992, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.72, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "7ceea6bf-3460-4644-908d-39a836ad8005", "answer_receipt_ids": [ "gen-1789921397-w0nvtTUEb1TdJENCDGue" ], "price_microusd": 5954, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "start_utc", "value": "10:00", "pass": false, "actual": null }, { "op": "eq", "field": "end_utc", "value": "10:30", "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-07", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 21, "started_at": "2026-09-20T16:23:14.628Z", "latency_ms": 9338.953582999995, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921401-1IHgb7WSR3tHWByO3Sh9" ], "price_microusd": 2977, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "unreserved_units", "value": 54, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 26, "started_at": "2026-09-20T16:23:35.696Z", "latency_ms": 10971.047332999995, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921424-eIrpIeKZx2SM3UAJO6Qx" ], "price_microusd": 3652, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-04", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 26, "started_at": "2026-09-20T16:23:46.668Z", "latency_ms": 8049.31941700005, "decision": { "lane": "balanced", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor" ], "confidence": 0.8, "policy_escalated": false, "risk_bump_applied": false, "high_stakes": false, "needs_tools": false, "needs_verification": true }, "first_lane": "balanced", "final_lane": "balanced", "fallback": false, "router_transaction_id": "c4886a57-e98f-4438-9a20-82a976e4cc90", "answer_receipt_ids": [ "gen-1789921434-EPmxwmTlx4B3bZ0m6sES" ], "price_microusd": 86, "charged_cents": 0, "answer": { "records_per_group": 56 }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "records_per_group", "value": 63, "pass": false, "actual": 56 } ] } }, { "repeat": 3, "prompt_id": "multi-step-03", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 34, "started_at": "2026-09-20T16:24:22.707Z", "latency_ms": 6511.251083999989, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921466-Su97MdXpPiWwbiJ2aZb3" ], "price_microusd": 4383, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "finish_minute", "value": 12, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-05", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 37, "started_at": "2026-09-20T16:24:26.057Z", "latency_ms": 12047.951957999961, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921475-obkATWMxekJf7CfB3mQM" ], "price_microusd": 3518, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "numerator", "value": 7, "pass": false, "actual": null }, { "op": "eq", "field": "denominator", "value": 13, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 44, "started_at": "2026-09-20T16:25:04.068Z", "latency_ms": 8499.624583999976, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921510-knPgZARuAUhZmUBOGIwY" ], "price_microusd": 2344, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 47, "started_at": "2026-09-20T16:25:09.990Z", "latency_ms": 13921.987333000056, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes", "low_confidence" ], "confidence": 0.56, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": false, "needs_verification": true }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "35005b29-6db8-4469-bfd9-1e858a497459", "answer_receipt_ids": [ "gen-1789921520-NOa5PIBBolj0KszhnZpX" ], "price_microusd": 4152, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "multi-step-01", "bucket": "multi-step", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 44, "started_at": "2026-09-20T16:25:12.569Z", "latency_ms": 13912.533708999981, "decision": { "lane": "frontier", "recommended": "balanced", "reason": [ "reasoning_tools_or_verification_floor", "low_confidence" ], "confidence": 0.58, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": false, "needs_tools": false, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "dff55918-7260-45d7-b434-8ddf350bb327", "answer_receipt_ids": [ "gen-1789921524-VYGQEFaBu3yNxHAwxDLz" ], "price_microusd": 2145, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "minutes", "value": 5, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "high-stakes-10", "bucket": "high-stakes", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 47, "started_at": "2026-09-20T16:25:23.913Z", "latency_ms": 10543.353082999936, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921530-Ubzqk70mf2LuoLyICnjA" ], "price_microusd": 4116, "charged_cents": 0, "answer": null, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "decision", "value": "HOLD", "pass": false, "actual": null }, { "op": "eq", "field": "human_review", "value": true, "pass": false, "actual": null } ] } }, { "repeat": 3, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "control", "run_id": "jev-20260920-50x3-metered", "sequence": 48, "started_at": "2026-09-20T16:25:26.482Z", "latency_ms": 9591.54483300005, "decision": null, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": null, "answer_receipt_ids": [ "gen-1789921534-7RFxCdeU0u8gRDowzarG" ], "price_microusd": 891, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } }, { "repeat": 3, "prompt_id": "tool-planning-08", "bucket": "tool-planning", "arm": "treatment", "run_id": "jev-20260920-50x3-metered", "sequence": 48, "started_at": "2026-09-20T16:25:36.074Z", "latency_ms": 4490.39383299998, "decision": { "lane": "frontier", "recommended": "fast", "reason": [ "reasoning_tools_or_verification_floor", "high_stakes" ], "confidence": 0.82, "policy_escalated": true, "risk_bump_applied": true, "high_stakes": true, "needs_tools": true, "needs_verification": false }, "first_lane": "frontier", "final_lane": "frontier", "fallback": false, "router_transaction_id": "62a0cf2b-843d-4205-a461-96f4c8d5e0f3", "answer_receipt_ids": [ "gen-1789921539-qrKE77MG6MgFkYf3zQqV" ], "price_microusd": 925, "charged_cents": 0, "answer": { "tool": "wallet.balance()" }, "grade": { "pass": false, "checks": [ { "op": "eq", "field": "tool", "value": "wallet.balance", "pass": false, "actual": "wallet.balance()" } ] } } ]