{"repeat":1,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:13:44.951Z","latency_ms":3717.676958,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920827-OF8hllNM95gyzOIhaW4Y"],"price_microusd":1334,"charged_cents":0,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":1,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:13:44.952Z","latency_ms":6083.547583,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920828-23m64hx3YHg5fBnfUk3f"],"price_microusd":1061,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:13:44.950Z","latency_ms":7761.841125000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.72,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0e7a7bde-d165-423d-a6cc-e3157784678f","answer_receipt_ids":["gen-1789920831-ygeDPbeDd9wpdwfPPKkP"],"price_microusd":1183,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":1,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:13:44.951Z","latency_ms":9438.125082999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.34,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"6ce22591-df4b-4021-b98d-51d2a04a6258","answer_receipt_ids":["gen-1789920832-eX62sSTisrmwGPW7FaCY"],"price_microusd":1587,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":1,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:13:52.713Z","latency_ms":5056.363708000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920836-BElO58NFaQxJznC0MJWq"],"price_microusd":1149,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":1,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:13:54.389Z","latency_ms":5018.662542,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920837-2qkY77YvuWPv5tYNjgNS"],"price_microusd":1551,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":1,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:13:48.670Z","latency_ms":11792.910750000001,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"f0039c6b-eb83-4273-a1f1-311bee618c83","answer_receipt_ids":["gen-1789920839-hK0m09EHaAaA7ClfrOPk"],"price_microusd":115,"charged_cents":0,"answer":{"text":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."}]}}
{"repeat":1,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:13:51.036Z","latency_ms":11154.099374999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.46,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"209052ae-7b5e-4a6d-a1a0-ec770bc5ae98","answer_receipt_ids":["gen-1789920840-XvMI47495Vk21OaTxijj"],"price_microusd":1097,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:14:02.191Z","latency_ms":3816.780332999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920844-RDXUnt3tkLw3aIPAZHf2"],"price_microusd":1396,"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."}]}}
{"repeat":1,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:13:57.770Z","latency_ms":9736.763125000001,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.82,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"c87a1774-f33d-4972-ac5d-6150364135d3","answer_receipt_ids":["gen-1789920846-32aSaUYRZyczMSUBFntf"],"price_microusd":106,"charged_cents":0,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":1,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:13:59.409Z","latency_ms":9732.520000000002,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.44,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a8b823af-e50a-4ca7-af9f-fceb6a84e7db","answer_receipt_ids":["gen-1789920847-zUg4iR3zjddyhUdY01LM"],"price_microusd":1173,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":1,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:14:00.464Z","latency_ms":10323.419208000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.32,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0f5201c9-e4f5-4767-83fe-ac97124a85f9","answer_receipt_ids":["gen-1789920849-fJy6j6AAA2NiP7IjRVPO"],"price_microusd":1056,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:14:07.508Z","latency_ms":6592.854292,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920851-1viTgPvRE78oocgg6rIM"],"price_microusd":3796,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:14:09.142Z","latency_ms":7605.490708000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920855-DL6NFlM2aS3UruHXipKV"],"price_microusd":1139,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":1,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:14:10.788Z","latency_ms":7623.683417,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920856-BQswWKgkWtZdvzrZsncF"],"price_microusd":1020,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:14:06.008Z","latency_ms":14578.815125000001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"cabde39e-4c8f-4998-b3bd-26d776b144ae","answer_receipt_ids":["gen-1789920858-HEbtBwmGjtE9DtuGaWnN"],"price_microusd":1430,"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."}]}}
{"repeat":1,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:14:14.101Z","latency_ms":8164.4986249999965,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920860-pBsh2GqZL3XZ6H2IuFS9"],"price_microusd":861,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":1,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:14:18.412Z","latency_ms":6068.956334000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920863-NfZfSdVv6B3MQVDBz9kK"],"price_microusd":1047,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:14:20.588Z","latency_ms":5507.775542000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920864-4GQORONG6H8nrGursPDF"],"price_microusd":639,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":1,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:14:16.748Z","latency_ms":13585.229041000002,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0923cd8e-b436-4b8b-8cf6-d44f0182e950","answer_receipt_ids":["gen-1789920868-RD7WV0K8vcmLddP9xy4d"],"price_microusd":2145,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:14:22.267Z","latency_ms":9990.503333,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"59f23bd5-6932-4751-aa16-f5bb4e27b83a","answer_receipt_ids":["gen-1789920871-6N8gV78Q4oCDXaxCD18U"],"price_microusd":51,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":1,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:14:30.334Z","latency_ms":5538.5453750000015,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920873-RpjQBLZkVZzbWXBCwN9z"],"price_microusd":2344,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:14:24.482Z","latency_ms":13079.187209000003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bd69465a-89ce-40d2-81ac-685a2e374c69","answer_receipt_ids":["gen-1789920876-QAeuGEVReMVcRUAg3kaf"],"price_microusd":1083,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:14:26.096Z","latency_ms":12171.656458999998,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e8e91239-6b95-4730-85f8-986bc64d1d1a","answer_receipt_ids":["gen-1789920877-XrQ51NSDu4W1MwUU1Fvd"],"price_microusd":46,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":1,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:14:32.258Z","latency_ms":7648.207999999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920878-iqHQEFDmZP6QzVkrSs13"],"price_microusd":1252,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":1,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:14:35.873Z","latency_ms":5637.163042,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920880-Gp38O8BwiJWcux2IN3vf"],"price_microusd":1087,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:14:37.562Z","latency_ms":8105.466666,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.39,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3812142e-7a8b-4f39-b5ab-f2843dd3bc64","answer_receipt_ids":["gen-1789920883-5hnCuq5mKpfkxNI718El"],"price_microusd":1106,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":1,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:14:38.269Z","latency_ms":11338.533583000004,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"655e099e-9fd0-4b23-ae3a-c314e6c37b02","answer_receipt_ids":["gen-1789920885-106k4Wy84pAn8o1MUytN"],"price_microusd":4152,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:14:39.907Z","latency_ms":11530.958707999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.67,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2db2dbe7-d04b-4d27-b4bd-7e5eff3ec736","answer_receipt_ids":["gen-1789920890-8PKxMvqxSyzp8dinaRIM"],"price_microusd":1288,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":1,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:14:41.511Z","latency_ms":11642.382916000002,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.31,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b12174a8-9649-46ac-aeed-9767a9284ed4","answer_receipt_ids":["gen-1789920891-mpjL3nRIDOqjMRvO0zOv"],"price_microusd":1123,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:14:45.668Z","latency_ms":9561.859915999994,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920893-vmzuAjEQabwjKg8hegIE"],"price_microusd":1072,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":1,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:14:49.608Z","latency_ms":9416.724624999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920895-VcP69x9O1rhZIevPEwQM"],"price_microusd":4116,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:14:51.439Z","latency_ms":12589.146375000011,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.85,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"8a86b237-9954-4b43-ab38-3473e8617b1a","answer_receipt_ids":["gen-1789920903-jvBs4UbgRfxIKbrJBHmI"],"price_microusd":82,"charged_cents":0,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":1,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:14:53.154Z","latency_ms":15047.419957999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.61,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"44ae8503-40ae-43f2-9b9d-c685aac31ab0","answer_receipt_ids":["gen-1789920906-Scz7n9FhhILzg6yXlW1Y"],"price_microusd":1046,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:14:55.230Z","latency_ms":14642.693459000002,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.43,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8c431dab-8843-4bc1-b95f-ffd3f28ef1a3","answer_receipt_ids":["gen-1789920908-1uN5Qslk6YCeNaRc5iQ9"],"price_microusd":1375,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":1,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:14:59.025Z","latency_ms":14519.380458,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.68,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"71855bb3-df35-4d2c-9f6f-16f26b558fd2","answer_receipt_ids":["gen-1789920910-EpcaWDUMl0P1XxHhUqDu"],"price_microusd":5671,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:15:04.029Z","latency_ms":11983.670125000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920913-cszYqG3oYE4SZK7AEBTy"],"price_microusd":2977,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:15:08.202Z","latency_ms":11202.886457999994,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920917-ukXKeulQmO3na3FL3Txg"],"price_microusd":1010,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:15:09.874Z","latency_ms":11224.28854200001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920919-KeFQqhkk6ABI8eQNvcIb"],"price_microusd":1339,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":1,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:15:13.545Z","latency_ms":11226.0055,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920921-QfTcra0Zv6sgDO1e0H1G"],"price_microusd":5918,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:15:16.013Z","latency_ms":11590.51933299999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920926-TkAF8BSMbMBlQbtrMI25"],"price_microusd":727,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":1,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:15:19.406Z","latency_ms":9997.600291999988,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920927-HCmxCq7jtGBeOS8Bf3GN"],"price_microusd":891,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":1,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:15:21.099Z","latency_ms":13849.611749999996,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d12f0327-6127-48a0-a0c7-f9a088c2f8c7","answer_receipt_ids":["gen-1789920932-ib3lzzzI2fvPHU1MhxS2"],"price_microusd":1847,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:15:24.772Z","latency_ms":11281.663707999993,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.85,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"e88a6d5c-d10f-48cc-9b27-bcc59bdab2ed","answer_receipt_ids":["gen-1789920935-eqVi02blmmOS8ThpCtl4"],"price_microusd":105,"charged_cents":0,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":1,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:15:27.604Z","latency_ms":9447.878167000003,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"a6df837d-c36f-4749-97cc-55846e24ec46","answer_receipt_ids":["gen-1789920936-tchUG8x2Wc32hrOxBDBy"],"price_microusd":48,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":1,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:15:29.404Z","latency_ms":8413.095291999998,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"21aa674a-a8fd-4d0e-90b4-d14684de6bc8","answer_receipt_ids":["gen-1789920937-2y9JQXe9el4YoGUN8bpO"],"price_microusd":51,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":1,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:15:34.950Z","latency_ms":5118.4441249999945,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920938-knJNG0BDVbRpqRjCOxsT"],"price_microusd":1813,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:15:36.054Z","latency_ms":6858.542083000008,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920940-y9uvDJVzL7TwqOknmIx0"],"price_microusd":3518,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:15:37.052Z","latency_ms":8756.409499999994,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920943-TyMdMOrL4veDegRJKQ1y"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:15:40.070Z","latency_ms":8095.431584000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920946-19u7kJ7X3G4lI0temawL"],"price_microusd":644,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":1,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:15:37.817Z","latency_ms":15598.847917000006,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.47,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5a45c3ed-f7b0-41dd-a318-2ba881753ca7","answer_receipt_ids":["gen-1789920949-oJxtcZ91D05hXg0rv81B"],"price_microusd":6074,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:15:42.913Z","latency_ms":14371.721125000011,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"efa21d0b-9f14-4d9f-b743-536e16bbcbaf","answer_receipt_ids":["gen-1789920955-b2PtoKnQIGTCaUQhpqQm"],"price_microusd":2179,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":1,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:15:45.809Z","latency_ms":14126.177291999993,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.93,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"2a99b9d8-911d-4d47-82ce-aaa54bb04f9a","answer_receipt_ids":["gen-1789920959-Ztcu8thDATLsL4D1K8Ju"],"price_microusd":96,"charged_cents":0,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":1,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:15:53.417Z","latency_ms":13072.538750000007,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920962-hBRlHGw3VlnWD2hcAVFE"],"price_microusd":6041,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:15:48.166Z","latency_ms":21598.310834000004,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"0fbb66a8-3717-4dff-867c-8a5efa7a885c","answer_receipt_ids":["gen-1789920969-MtmpwgAhPYEQHj5VEikj"],"price_microusd":47,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":1,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:15:57.286Z","latency_ms":15211.780208000011,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920970-Xlc1gK9bfT03NTAGaYqN"],"price_microusd":2117,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":1,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:16:06.490Z","latency_ms":8216.185375000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920973-XNlSn0TJ4TlZhxT8DYIn"],"price_microusd":1123,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":1,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:16:09.766Z","latency_ms":7663.948499999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920974-NAuvsl9nP29ykgVHfa30"],"price_microusd":3652,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:16:12.499Z","latency_ms":7949.477958000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920979-LkfnfS8X2FIQQdegR96g"],"price_microusd":732,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":1,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:15:59.936Z","latency_ms":22227.362416999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.3,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3e600042-a46b-49a8-a1f2-838bb79661ae","answer_receipt_ids":["gen-1789920980-vgoSLftSmJM7IyoHysHS"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:16:22.157Z","latency_ms":3789.354040999984,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920984-AnYB6zvTikrKh1meSGIQ"],"price_microusd":964,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:16:14.707Z","latency_ms":13061.538083000021,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"f65d1380-5494-4498-8fe3-6c8e34bb9272","answer_receipt_ids":["gen-1789920986-zuUtO7lmNmUTv8LRaYU1"],"price_microusd":1157,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":1,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:16:17.430Z","latency_ms":12412.298500000004,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.8,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"ba084737-7917-4463-9397-2e571cc7f618","answer_receipt_ids":["gen-1789920989-HEkjGqfAP8BFH1oPhP41"],"price_microusd":86,"charged_cents":0,"answer":{"records_per_group":56},"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":56}]}}
{"repeat":1,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:16:20.445Z","latency_ms":10613.203957999998,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"efc1effc-309f-4bbd-9a39-7de03e13f265","answer_receipt_ids":["gen-1789920990-PPe90C8Ez6M38yjAdJrB"],"price_microusd":47,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":1,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:16:25.943Z","latency_ms":6859.443165999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920991-aWKZjaYbz9cDJdg1r8j3"],"price_microusd":1164,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":1,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:16:27.757Z","latency_ms":6848.032124999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":[null],"price_microusd":0,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:16:29.829Z","latency_ms":7189.18766700002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920994-DtMCYpAGJd2HMZTEINI4"],"price_microusd":773,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":1,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:16:31.049Z","latency_ms":8613.275708000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789920997-u00EyXM2FanQRIvz5mi2"],"price_microusd":1118,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":1,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:16:32.799Z","latency_ms":12650.574083999993,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.46,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1b919db8-ffe8-4030-ba27-e0620c17a1d4","answer_receipt_ids":["gen-1789921002-Ar9peSmgLbDN4b9Ot6Pt"],"price_microusd":1198,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":1,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:16:34.602Z","latency_ms":12367.679334000015,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b981b617-29d4-49d4-930c-21d2aa87f3c1","answer_receipt_ids":["gen-1789921005-P4WWbV7RHIBB3EhM8xT5"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:16:37.015Z","latency_ms":11102.216832999984,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"dc7d34c5-3d73-4d18-8edc-7f7b3fcf0042","answer_receipt_ids":["gen-1789921007-JfLLq4ygKehsSnd8b2lj"],"price_microusd":49,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":1,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:16:39.660Z","latency_ms":9951.772041999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.6,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b3bbfa07-77e4-46b4-bd98-fec589a7e28c","answer_receipt_ids":["gen-1789921008-7MfVHeqYaDCgBdGgg4fO"],"price_microusd":1152,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":1,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:16:45.446Z","latency_ms":5747.283208000008,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921009-7Se6g1D3nUHRrUsrUQTN"],"price_microusd":1360,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":1,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:16:46.966Z","latency_ms":5900.592791999981,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921011-H3BTbKzhrgHhl3GqWHZg"],"price_microusd":691,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":1,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:16:49.610Z","latency_ms":5905.642667000007,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921013-RSk6aSfFuMYW1zhlKUtp"],"price_microusd":1417,"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":1,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:16:48.114Z","latency_ms":11803.25870799998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.76,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"aba6697d-c50d-4206-bf10-37f49d95eabb","answer_receipt_ids":["gen-1789921018-mQGKkJhFxLc5ybjs5PJU"],"price_microusd":1317,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":1,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:16:51.193Z","latency_ms":10979.594417000015,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.64,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"520e7aad-591b-4a7e-922d-9e969933f775","answer_receipt_ids":["gen-1789921020-cTs4Qx0RaXl6IGnP7wje"],"price_microusd":1396,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":1,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:16:52.866Z","latency_ms":11614.467000000004,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.96,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"42cb7bc2-ab9e-4f75-ae5d-33005d99e0dc","answer_receipt_ids":["gen-1789921024-Z7mDaU4Zy7Y01Uc87y5Z"],"price_microusd":47,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":1,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:16:59.916Z","latency_ms":6324.888957999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921024-gvRaVHnCAUdn4S6OZIgE"],"price_microusd":1283,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":1,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:16:55.516Z","latency_ms":12470.365250000003,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.38,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9051ab90-a1d5-423c-8ba3-9ea52cd4e999","answer_receipt_ids":["gen-1789921026-Hcx4XG3YeqCaQO6f2x4y"],"price_microusd":1476,"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. The owner is Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. The owner is Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. The owner is Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. The owner is Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. The owner is Rina."}]}}
{"repeat":1,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:17:02.172Z","latency_ms":7785.107250000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921028-rJBKHFZBLp4dWQpK1Zil"],"price_microusd":1206,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":1,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:17:04.480Z","latency_ms":7109.833916000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921030-2fOmc5ehDmirXz1QRSKJ"],"price_microusd":891,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":1,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:17:07.986Z","latency_ms":6229.562624999991,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921032-lWfahXjEQGszyeZMm561"],"price_microusd":948,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:17:06.241Z","latency_ms":13541.285083000024,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.78,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0fc9a91c-3b55-420c-b3ea-b91d6d432f45","answer_receipt_ids":["gen-1789921037-E4AQdni8bFJrxAXlcai2"],"price_microusd":2584,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:17:09.957Z","latency_ms":12304.334499999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.3,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"242f6c12-e796-472d-a3f0-772834917dce","answer_receipt_ids":["gen-1789921040-Gm439dv6ntxJNrzsM4pW"],"price_microusd":1240,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":1,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:17:11.589Z","latency_ms":12131.679457999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes"],"confidence":0.84,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5b9a0cc3-3c18-4caf-97aa-4c62fde7ba2a","answer_receipt_ids":["gen-1789921042-KLEM5lCMqfvdgaZGYijM"],"price_microusd":925,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":1,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:17:19.782Z","latency_ms":6247.611540999991,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921043-BHa7TXLNuzR8KFMF8B6o"],"price_microusd":2550,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:17:14.216Z","latency_ms":13448.367499999993,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.45,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"22646e02-9988-4ff0-a8ca-e50b3bac8ad8","answer_receipt_ids":["gen-1789921046-JEMIVZbtBhZ2NtYC23Br"],"price_microusd":982,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":1,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:17:22.262Z","latency_ms":6982.259250000003,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921047-MrHOoKgxLHy1wfGRIicI"],"price_microusd":557,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:17:23.721Z","latency_ms":8708.323082999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921049-UzurIRJLhpfmAEZxoT1n"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":1,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:17:26.030Z","latency_ms":8398.02774999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921052-XEit7xSldBLmm1z2k7xQ"],"price_microusd":1381,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":1,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:17:27.664Z","latency_ms":10926.141374999977,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.47,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"7f6741ce-dc0b-43e1-8b5e-e2fb8489d653","answer_receipt_ids":["gen-1789921057-Qg5Z2YYUtewqPvU8zNxk"],"price_microusd":1221,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:17:29.244Z","latency_ms":10167.80454099999,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"89b6ab16-8b61-4677-973b-37e2757f6130","answer_receipt_ids":["gen-1789921058-AZOTtxmm1SFya0dnaEny"],"price_microusd":44,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:17:32.430Z","latency_ms":7933.507541999978,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.85,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"a4bc8106-be0c-4311-a467-38118c44f419","answer_receipt_ids":["gen-1789921059-SMTdtYhh5Cw80p5ayiFY"],"price_microusd":92,"charged_cents":0,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":1,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:17:34.429Z","latency_ms":8003.1437500000175,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.55,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a3cff623-e10f-4f2c-8228-57c7d228b7d5","answer_receipt_ids":["gen-1789921060-P4V9IieXHDxa6f6WQChk"],"price_microusd":1415,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":1,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:17:38.591Z","latency_ms":7090.321250000008,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921063-5n2ebgWIMHJOqhOrk03v"],"price_microusd":1185,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":1,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:17:39.412Z","latency_ms":9178.663500000024,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1b39a581-4869-4912-a865-4ffbbe14a0cb","answer_receipt_ids":["gen-1789921067-IorROgIEY7pRQZ5nR3pO"],"price_microusd":931,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":1,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:17:40.364Z","latency_ms":8969.357958000008,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.93,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"a22f4a91-95d9-4a9e-baa6-735dba7dc98e","answer_receipt_ids":["gen-1789921068-t1GEEpVGzyVuphHHkjpu"],"price_microusd":49,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":1,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:17:48.592Z","latency_ms":2432.359582999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921069-nfT4iIJKFENoBl0xKiyf"],"price_microusd":897,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":1,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:17:49.334Z","latency_ms":3344.7253330000094,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921071-gDsW3fOnOx3WMzbSA9LZ"],"price_microusd":773,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:17:52.810Z","latency_ms":3518.14175000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921075-jJU4DOeuo9uzgDwPn0DA"],"price_microusd":557,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:17:52.809Z","latency_ms":5124.889332999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.39,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c7bca728-7c73-4e3a-b11b-9b3c1219193a","answer_receipt_ids":["gen-1789921076-hvZYjK0THq1haZXPWFts"],"price_microusd":1106,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":2,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:17:52.809Z","latency_ms":6764.063000000024,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1c40dc54-cf56-443b-96db-55337377ce52","answer_receipt_ids":["gen-1789921078-kDxgDW2bTk3RcrsqkanK"],"price_microusd":1173,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":2,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:17:52.810Z","latency_ms":8560.940583000018,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.31,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1854cfbe-ddc4-4599-a154-4852dce2f142","answer_receipt_ids":["gen-1789921079-UWKQ65jhZQduBLE8UBTI"],"price_microusd":1123,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:17:57.934Z","latency_ms":5807.573082999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921082-a7CFEiuZZjmSnicgCvrT"],"price_microusd":1072,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":2,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:17:59.574Z","latency_ms":5635.926084000006,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921083-08snqmq5wD8BLA2hEPN1"],"price_microusd":1139,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":2,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:18:01.371Z","latency_ms":5288.545583999978,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921085-WfBL8KFvrJTQKXETHaPQ"],"price_microusd":1087,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:17:56.328Z","latency_ms":11070.635332999984,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"300d940e-27f8-4c55-9bd5-279e970596ff","answer_receipt_ids":["gen-1789921086-BqsZjoXdbGeZ4StFvLF6"],"price_microusd":44,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:18:05.210Z","latency_ms":4547.647874999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921088-Ni3u0LLvHUK4u25igTUT"],"price_microusd":1252,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":2,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:18:06.660Z","latency_ms":4839.899875000032,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921090-h2S3RABDFEUBInBUKOrU"],"price_microusd":1551,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":2,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:18:03.742Z","latency_ms":9495.90849999999,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.83,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"9bacd833-7994-49c1-9aa5-007f3785c77b","answer_receipt_ids":["gen-1789921092-t37s7OmvKbb50CNv4hg8"],"price_microusd":81,"charged_cents":0,"answer":{"seconds":20},"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":20}]}}
{"repeat":2,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:18:07.400Z","latency_ms":8789.733707999985,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1fcd3d6b-dff1-491c-b83c-8da8e7de4da5","answer_receipt_ids":["gen-1789921094-4AVCoaclmKv3lXCfSMoj"],"price_microusd":1083,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:18:13.238Z","latency_ms":5191.205083000008,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921096-k94gMAWnIrsqYCWkzX0z"],"price_microusd":2550,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:18:09.758Z","latency_ms":10336.689249999996,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.69,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4667eb02-4fd1-4ee5-bd78-dbf11eb4c3e8","answer_receipt_ids":["gen-1789921098-zsD7bjU8Mhw7i177iDPV"],"price_microusd":1288,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":2,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:18:11.501Z","latency_ms":10338.455791999993,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.4,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bf0d52b0-dd84-4236-875d-e474d70a38dd","answer_receipt_ids":["gen-1789921100-QaGUzpomv0o8dUpkob0G"],"price_microusd":1587,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":2,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:18:16.190Z","latency_ms":7182.797999999952,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921102-wKvrHxLg23e7C2NCjA1l"],"price_microusd":1047,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:18:18.430Z","latency_ms":6894.752458000032,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921103-DdtWTpxAn2RH7gWtKUOA"],"price_microusd":691,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":2,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:18:21.840Z","latency_ms":6143.71979199996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921106-LZICYwlah0rBFO764DUX"],"price_microusd":1417,"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":2,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:18:20.095Z","latency_ms":9777.596374999965,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.87,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"bd3f0183-7636-41e6-b91f-d0d67e280688","answer_receipt_ids":["gen-1789921109-5DJ7fzLQ3f1t4L3IA6vG"],"price_microusd":49,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:18:23.374Z","latency_ms":8935.846958999988,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.26,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a52c026b-3888-4b17-92df-b26eae8f4c7c","answer_receipt_ids":["gen-1789921110-WlNY7HQywKPej1jI1ycx"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:18:25.325Z","latency_ms":7878.8310830000555,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.94,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"a92ad857-7156-4362-b2a1-8fba0f8839bf","answer_receipt_ids":["gen-1789921112-PmFCCNkifCbNBMJFoDE7"],"price_microusd":47,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":2,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:18:29.873Z","latency_ms":4820.168416999979,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921113-idcjF3NosmAh2q1xdC9s"],"price_microusd":773,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":2,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:18:27.984Z","latency_ms":8483.055000000051,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.44,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c74c390a-0ded-455d-8cf0-04b69756d6c1","answer_receipt_ids":["gen-1789921114-FVdXlUgKCcvZgTr32DM7"],"price_microusd":1373,"charged_cents":0,"answer":{"text":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."}]}}
{"repeat":2,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:18:32.310Z","latency_ms":6366.09833399998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921116-clB4wTHrUkNRfrDHPq4l"],"price_microusd":964,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:18:33.204Z","latency_ms":8288.533250000037,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921118-iihPcxOxwfWGPQT53jNn"],"price_microusd":1206,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":2,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:18:34.694Z","latency_ms":9289.363249999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921121-f90ZDqcUHqXFTx1KdgOE"],"price_microusd":2344,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:18:36.468Z","latency_ms":11380.866625000024,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921124-DHMNaZ6TQNhFVDqFXWSW"],"price_microusd":5635,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:18:38.676Z","latency_ms":15907.897665999946,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.76,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"7f6d6c98-81f3-4b11-bb1a-577c8406391f","answer_receipt_ids":["gen-1789921132-cciZkIIyfEPRh9lRgqbF"],"price_microusd":931,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":2,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:18:41.493Z","latency_ms":15034.646999999997,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.3,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"aee9fb1e-e524-49eb-a607-0de04df7aaf5","answer_receipt_ids":["gen-1789921134-4zUfEO1wUa4poIGECTbB"],"price_microusd":1240,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":2,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:18:43.984Z","latency_ms":15459.498916000011,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.61,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"854a7a3f-e9a7-4446-a75d-49051ac5500b","answer_receipt_ids":["gen-1789921136-HCnDCSkQuJnIfwq83Nqm"],"price_microusd":2145,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:18:47.849Z","latency_ms":15147.289667000005,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.72,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5c6d7a2b-a8d5-47bc-9041-6de4a5ec1203","answer_receipt_ids":["gen-1789921139-56uOcmEUxAIgh1opEgdn"],"price_microusd":5954,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:18:54.584Z","latency_ms":10259.117542000022,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921143-vAC3Ys5sgZ4XppJcq8uo"],"price_microusd":897,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":2,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:18:56.529Z","latency_ms":9757.590124999988,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921145-RAzUKzGUG6L4nivcV7dq"],"price_microusd":891,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":2,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:19:02.997Z","latency_ms":5737.342582999961,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921147-vzSUZIB812kgpCsvIz1s"],"price_microusd":1360,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":2,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:18:59.444Z","latency_ms":12132.792750000022,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.43,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4d8ed963-3d12-4a76-b139-cce05993755a","answer_receipt_ids":["gen-1789921150-CmFuTHAzzPigauS0dAQG"],"price_microusd":1097,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:19:04.844Z","latency_ms":9527.281041999988,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c254ab88-3808-4140-bcf1-cab40471e6cb","answer_receipt_ids":["gen-1789921152-uvC4PwciCw4RBiPehTq8"],"price_microusd":1198,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":2,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:19:06.286Z","latency_ms":9721.519582999987,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes"],"confidence":0.85,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ee4525ab-302c-4110-955e-6ba0ef0ea41a","answer_receipt_ids":["gen-1789921154-E6T32zMswFkqaW3QxsXe"],"price_microusd":925,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":2,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:19:11.578Z","latency_ms":6065.870500000019,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921156-sahpKa0MXSTVulP2CjXP"],"price_microusd":1061,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:19:08.735Z","latency_ms":10651.358290999953,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.66,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8fdc5e90-2593-4be6-915a-94a362cd8a8e","answer_receipt_ids":["gen-1789921157-jiab0Up1VdYQpmqbVQhk"],"price_microusd":1396,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":2,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:19:14.371Z","latency_ms":6766.027375000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921159-gsSQaeqnaZ00WKxSat7D"],"price_microusd":1164,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":2,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:19:17.644Z","latency_ms":5702.113292000024,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921161-TCYgbbrzpYldp0LgU05i"],"price_microusd":861,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":2,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:19:19.387Z","latency_ms":5637.411416999996,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921163-2ouV2kODItY2HqFCPRsr"],"price_microusd":1149,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":2,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:19:16.008Z","latency_ms":11201.198957999994,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2f10d8b8-9e74-4efb-9702-b83c92874167","answer_receipt_ids":["gen-1789921165-I6Ducit7EcET929YvqvO"],"price_microusd":1157,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":2,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:19:21.138Z","latency_ms":9512.409125000006,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.27,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"5d3bad25-84bd-44c0-a9ad-5b829559abad","answer_receipt_ids":["gen-1789921168-z9IFLfdAWrOfvn3cXwnT"],"price_microusd":1056,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:19:27.210Z","latency_ms":5181.989499999967,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921170-uDLQGpSAll7qiQ8Xj8Ix"],"price_microusd":1123,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":2,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:19:23.347Z","latency_ms":9863.940416999976,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"c998d534-0642-42b1-9881-ba0a045086c6","answer_receipt_ids":["gen-1789921172-yon2kGO0OpVLi4vMJY6T"],"price_microusd":51,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":2,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:19:25.025Z","latency_ms":9789.067624999967,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.69,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"db53ea55-93bc-4d87-b14e-1d0952a83e3f","answer_receipt_ids":["gen-1789921173-MmEb6urNUY0zQLAGPWz9"],"price_microusd":1183,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":2,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:19:30.651Z","latency_ms":5785.527042000031,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921175-bQy8BtdF3nxdMqOhuaKN"],"price_microusd":1020,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:19:32.393Z","latency_ms":7619.620750000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921176-m1qhRyuXSJd4YNBDZCkt"],"price_microusd":6041,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:19:34.814Z","latency_ms":8882.794166000036,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921181-YEKLDBdTWvIIlHEnfpjV"],"price_microusd":3796,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:19:33.211Z","latency_ms":13728.110207999998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.6,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"59c3a478-ec02-4813-b14a-5b80a2adaafb","answer_receipt_ids":["gen-1789921184-H6BZdlsrMmnyKMJA8vGC"],"price_microusd":1847,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":2,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:19:36.438Z","latency_ms":13089.548917000007,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.6,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8ac3f9b7-1979-4246-a322-1b18c3391142","answer_receipt_ids":["gen-1789921187-T4c5visk6keUhkuns1sk"],"price_microusd":1046,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:19:40.013Z","latency_ms":13460.186000000045,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.37,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"ed05bdc6-8327-4598-afe0-65961fde7045","answer_receipt_ids":["gen-1789921189-pcutvrKzCywrzUDPdToD"],"price_microusd":6074,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:19:46.940Z","latency_ms":8828.939541,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921193-PwiYA3jnFAwAewddEIlD"],"price_microusd":1813,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring service and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:19:43.698Z","latency_ms":13113.887790999957,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.85,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"233981ec-841b-48e4-867b-d09918f1b6f5","answer_receipt_ids":["gen-1789921196-mKUOLBbH6NdTCeEtmyCD"],"price_microusd":106,"charged_cents":0,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":2,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:19:49.528Z","latency_ms":9308.435375000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921197-60TaGAX7UsoEs7VsoQy7"],"price_microusd":1010,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:19:56.812Z","latency_ms":7196.894042,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921200-sMBNe1ZjehV3LrIcsoF2"],"price_microusd":4116,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:19:53.474Z","latency_ms":13217.946875000023,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.54,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3cb72333-3274-42bd-9c1f-f40114f63056","answer_receipt_ids":["gen-1789921205-FSPyRc4ePcThXnnBAx1I"],"price_microusd":1221,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:19:55.769Z","latency_ms":12782.456958000024,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.43,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2221ee5b-3088-40c9-a2fd-12969073398d","answer_receipt_ids":["gen-1789921207-hubE2Hz8d7zOi2Ccm4RB"],"price_microusd":1375,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":2,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:19:58.837Z","latency_ms":12602.219249999966,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.73,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9245f2b6-70a2-48ee-8ed9-044f4e839382","answer_receipt_ids":["gen-1789921209-lTH7zM4dWWLGaaT68Xja"],"price_microusd":1317,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":2,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:20:06.692Z","latency_ms":6557.8718339999905,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921211-mEZIC3iD6to9YNyzTA7l"],"price_microusd":1185,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":2,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:20:08.552Z","latency_ms":6317.413667000015,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921213-BQFgbv1YTnO9y4AdBgKd"],"price_microusd":1339,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":2,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:20:04.010Z","latency_ms":14705.216749999963,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4b89c2be-ae66-4912-bc96-6cb611a46f0c","answer_receipt_ids":["gen-1789921215-IQ5opdeQcMuge9pi212M"],"price_microusd":4152,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:20:11.440Z","latency_ms":9135.142541999987,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921218-SgVa4Rbp6XdoWCajSpk4"],"price_microusd":1283,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":2,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:20:13.251Z","latency_ms":9976.377916999976,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921220-d5pGwaleFPqflM6Pu5S2"],"price_microusd":3652,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:20:20.575Z","latency_ms":6700.311625000031,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921224-5Xw30kKpF3bPo9Tc7XMv"],"price_microusd":3518,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:20:14.870Z","latency_ms":14635.152583000017,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.62,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0f66e772-619e-4552-9ae3-ea37372b679f","answer_receipt_ids":["gen-1789921228-uv4EgMGuWd1EjcBYP5yK"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:20:18.715Z","latency_ms":13684.21762499999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.78,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e01b4426-0f40-4085-a04f-5ea07c62ee84","answer_receipt_ids":["gen-1789921229-2lb4ABOrhpwmvzibLAN9"],"price_microusd":2153,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated estimated arrival of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:20:23.227Z","latency_ms":10881.372415999998,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.81,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"47ffa13a-2950-41b1-8db6-05131014fd9c","answer_receipt_ids":["gen-1789921233-knAx64XEFp6n8AQ99b29"],"price_microusd":86,"charged_cents":0,"answer":{"records_per_group":56},"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":56}]}}
{"repeat":2,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:20:29.506Z","latency_ms":6271.80041700002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921234-3a2sEWxJwwAZpIvIfS2x"],"price_microusd":964,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:20:32.400Z","latency_ms":6092.613958000031,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921236-GlanH4k4Hcyzs4dLniad"],"price_microusd":2143,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":2,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:20:27.276Z","latency_ms":12493.087415999966,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.83,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"74153213-40e5-47e7-a9bb-3280867fd444","answer_receipt_ids":["gen-1789921238-T2djzRmSCarvgzoUkYfr"],"price_microusd":105,"charged_cents":0,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":2,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:20:35.778Z","latency_ms":6564.191416000016,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921240-DCOgjCkeeo2viUM9EVRz"],"price_microusd":1370,"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."}]}}
{"repeat":2,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:20:39.769Z","latency_ms":4864.539583000005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921243-lHXltgSrX2hcs3tBE7KM"],"price_microusd":948,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:20:34.109Z","latency_ms":11418.792541000003,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e74dc57a-7d9f-4ab8-b02f-69ee9fc1514e","answer_receipt_ids":["gen-1789921245-gjkfBP62ipp2NiQbeHv5"],"price_microusd":51,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":2,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:20:38.493Z","latency_ms":8341.239500000025,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"6879f873-d072-47d5-9253-296c121894a7","answer_receipt_ids":["gen-1789921246-ivTDW3vGtTdrQkKhyMOG"],"price_microusd":82,"charged_cents":0,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":2,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:20:45.529Z","latency_ms":3557.944583000033,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921247-WqrJbdwosHRHuVzqYhw4"],"price_microusd":891,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":2,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:20:42.343Z","latency_ms":8997.536334000004,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.28,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"2787a4a3-12f8-419a-af2b-2fa67f9f1a49","answer_receipt_ids":["gen-1789921249-OiRNN3QCuCQ4nHQGj7g0"],"price_microusd":1430,"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, simply request a new link to reset your password."}]}}
{"repeat":2,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:20:46.835Z","latency_ms":9940.108542000002,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921254-hL915nixRrgWfREnQu9x"],"price_microusd":2977,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:20:44.635Z","latency_ms":15219.969166999974,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"57212120-567b-4757-85d4-59b2291ca6ff","answer_receipt_ids":["gen-1789921258-NJm7sgdQ2LnJTMpiDHPv"],"price_microusd":982,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":2,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:20:49.088Z","latency_ms":12297.26800000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921260-8LJNC9gMIg898xtHjiBP"],"price_microusd":639,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":2,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:20:56.775Z","latency_ms":7162.238083000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921262-mBs66YFTp1X3lwtAOCwQ"],"price_microusd":727,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":2,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:20:51.341Z","latency_ms":15468.793875000032,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.68,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4ceecc48-0067-4a6e-8562-f4d6051b032a","answer_receipt_ids":["gen-1789921265-IzweCAPpIenSIBA1GivM"],"price_microusd":1152,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":2,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:20:59.855Z","latency_ms":8605.331000000006,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.83,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"12432133-8dc0-40d7-ba96-727898b9238b","answer_receipt_ids":["gen-1789921267-AWNDpVRJUqrw4cxpwHFV"],"price_microusd":92,"charged_cents":0,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":2,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:21:01.386Z","latency_ms":7750.072291000048,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"55c779c0-99da-4b8c-8426-8571d4e0428e","answer_receipt_ids":["gen-1789921268-eKsEGfDeT9knW2VaJEuD"],"price_microusd":46,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":2,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:21:06.810Z","latency_ms":3820.4905000000144,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921269-fJOrg0nsVEQZKR6B4CFR"],"price_microusd":1118,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":2,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:21:03.938Z","latency_ms":7449.490665999998,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.88,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"2fcdd5db-bf90-46c6-bdd3-03c3717e6551","answer_receipt_ids":["gen-1789921270-6hPv3DstMWK1TIWMHof2"],"price_microusd":48,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":2,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:21:08.460Z","latency_ms":6331.512792000023,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921271-1H0fjPEJAlcWXBFfK33E"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:21:09.137Z","latency_ms":7805.5740839999635,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921275-YG6S3uHwAdVqvK2qcBke"],"price_microusd":1381,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":2,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:21:10.631Z","latency_ms":8544.853042000032,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921277-quIwXr7RAjFXyzezHNsf"],"price_microusd":1334,"charged_cents":0,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":2,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:21:11.389Z","latency_ms":13334.790749999986,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"b6f03351-5964-4e88-be31-b7eaa61dec48","answer_receipt_ids":["gen-1789921284-dTKcQVIzxtHfwOXhnC8J"],"price_microusd":47,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":2,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:21:14.792Z","latency_ms":11163.057583999995,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.92,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"0699792d-0954-4b6b-a946-fd1399fbf413","answer_receipt_ids":["gen-1789921284-yd3lJH4bps9SXzXMfe0Y"],"price_microusd":96,"charged_cents":0,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":2,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:21:16.943Z","latency_ms":11475.16966700001,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.53,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"62d4da4d-68c5-4775-8464-f26f82ef1b55","answer_receipt_ids":["gen-1789921286-UHU0r9SUH4AXEWeFfQgY"],"price_microusd":1415,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":2,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:21:19.176Z","latency_ms":10976.078582999995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.28,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b22ada6c-e28e-48c2-8c7c-363551e0f625","answer_receipt_ids":["gen-1789921288-BXrCYS0WOUjvmXrr3li8"],"price_microusd":1368,"charged_cents":0,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":2,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:21:24.724Z","latency_ms":8295.666209000046,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921291-piHPGQCz4AgcCP9B9mKP"],"price_microusd":644,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":2,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:21:25.956Z","latency_ms":11379.240417000023,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921294-2GEWCghZO0awxrws68x3"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":2,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:21:28.418Z","latency_ms":10948.433375000022,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"ca72f51b-a5c1-48cf-811c-92002a4e2c2e","answer_receipt_ids":["gen-1789921298-P4NkVD03zOAlcKB2cxb3"],"price_microusd":49,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":2,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:21:30.153Z","latency_ms":9927.658292000007,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"009fc976-9903-480b-b48b-2a824380541e","answer_receipt_ids":["gen-1789921299-erKU6SnAeXTYGeQzhSfl"],"price_microusd":47,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":2,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:21:39.367Z","latency_ms":2413.6803329999675,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921300-1Hq3o1SxHwFO1Hw9AA4I"],"price_microusd":773,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":2,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:21:40.082Z","latency_ms":3690.073708000011,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921302-caAHlEWNkDi9MzWcEnPU"],"price_microusd":732,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:21:43.924Z","latency_ms":2759.906541000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921305-odFWpMcgja2oOkmD7T6n"],"price_microusd":1360,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":3,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:21:43.924Z","latency_ms":6556.170582999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921306-LTwHBu7anbofL19CU3AD"],"price_microusd":6041,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:21:43.922Z","latency_ms":9647.270499999984,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"f4cda76a-f578-4dc3-bc18-2e522f30d737","answer_receipt_ids":["gen-1789921312-Rw01TDNcay2bXpbKD1zK"],"price_microusd":1198,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":3,"prompt_id":"lookup-08","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:21:43.923Z","latency_ms":10321.375375000003,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.92,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"20b90cff-1e70-4f2f-81f1-497bc7402240","answer_receipt_ids":["gen-1789921313-zq95iwo9bftob48slIaf"],"price_microusd":47,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":3,"prompt_id":"rewriting-08","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":0,"started_at":"2026-09-20T16:21:53.570Z","latency_ms":3632.615375000052,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921315-OVEju1QbZbwKNIPgaWKO"],"price_microusd":1164,"charged_cents":0,"answer":{"text":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"24","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"October 2","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"contains","field":"text","value":"laptop","pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"The workshop has 24 seats. Registration closes October 2. Bring a laptop."}]}}
{"repeat":3,"prompt_id":"lookup-08","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":1,"started_at":"2026-09-20T16:21:54.245Z","latency_ms":6013.777084000001,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921318-z2Qr3fxxbGFo2sHE2BFy"],"price_microusd":691,"charged_cents":0,"answer":{"expansion":"recovery point objective"},"grade":{"pass":true,"checks":[{"op":"eq","field":"expansion","value":"recovery point objective","pass":true,"actual":"recovery point objective"}]}}
{"repeat":3,"prompt_id":"tool-planning-02","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":2,"started_at":"2026-09-20T16:21:46.684Z","latency_ms":15359.868791999994,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.68,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0e1a27d1-0f1d-4b26-8b87-0f6e3c09b44e","answer_receipt_ids":["gen-1789921320-hWcMyVqoz8Zynam1UW1r"],"price_microusd":1396,"charged_cents":0,"answer":{"tool":"github.get_issue","repo":"acme/demo","number":42},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.get_issue","pass":true,"actual":"github.get_issue"},{"op":"eq","field":"repo","value":"acme/demo","pass":true,"actual":"acme/demo"},{"op":"eq","field":"number","value":42,"pass":true,"actual":42}]}}
{"repeat":3,"prompt_id":"multi-step-08","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":3,"started_at":"2026-09-20T16:21:50.481Z","latency_ms":15488.636457999994,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"e299c13f-28c9-4f3c-98b8-b6fa56272ab9","answer_receipt_ids":["gen-1789921322-c4S5xNAhrTSUyvvRE9MU"],"price_microusd":6074,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":26,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:21:57.204Z","latency_ms":10323.641917,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921326-GCUiZM7MC1Wi7ukzf5zt"],"price_microusd":1185,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"lookup-01","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:22:02.044Z","latency_ms":7840.622959,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921328-UkL09xKwHXQvRAacounR"],"price_microusd":727,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":3,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:22:00.259Z","latency_ms":12390.213749999995,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"36de3104-b90e-4e02-a000-29bcdc301f40","answer_receipt_ids":["gen-1789921331-OPZmFwShLGig7Tvc0UtV"],"price_microusd":1152,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":3,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:22:05.970Z","latency_ms":10222.091415999981,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.74,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bad107af-b7b4-4e08-b867-87fda79cfa3b","answer_receipt_ids":["gen-1789921334-EJE7TaUpQd3s72m3LYMq"],"price_microusd":1317,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-09","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":4,"started_at":"2026-09-20T16:22:07.528Z","latency_ms":10284.655957999988,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.5,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"312e6111-d88c-482f-9d97-1955818720c4","answer_receipt_ids":["gen-1789921336-mz8hZTiqGtgqU2XkNAnm"],"price_microusd":1221,"charged_cents":0,"answer":{"decision":"GRANT_READ_ONLY","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"GRANT_READ_ONLY","pass":true,"actual":"GRANT_READ_ONLY"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"tool-planning-01","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":5,"started_at":"2026-09-20T16:22:12.650Z","latency_ms":6929.512832999986,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921338-CYQiK9eIXtLIZacNUejg"],"price_microusd":1118,"charged_cents":0,"answer":{"tool":"catalog.lookup","product_id":"P-17"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"catalog.lookup","pass":true,"actual":"catalog.lookup"},{"op":"eq","field":"product_id","value":"P-17","pass":true,"actual":"P-17"}]}}
{"repeat":3,"prompt_id":"lookup-01","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":6,"started_at":"2026-09-20T16:22:09.885Z","latency_ms":10385.480625000026,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"13fd4183-05cc-43c9-8152-85366fc3614f","answer_receipt_ids":["gen-1789921339-Rk6hJWH0DYnlxyENaVo7"],"price_microusd":48,"charged_cents":0,"answer":{"product":"Cedar","quantity":0},"grade":{"pass":true,"checks":[{"op":"eq","field":"product","value":"Cedar","pass":true,"actual":"Cedar"},{"op":"eq","field":"quantity","value":0,"pass":true,"actual":0}]}}
{"repeat":3,"prompt_id":"rewriting-02","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":7,"started_at":"2026-09-20T16:22:16.193Z","latency_ms":5908.048999999999,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921340-AjiQ1j46dVjU3Sg9jwqA"],"price_microusd":1283,"charged_cents":0,"answer":{"text":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Friday","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"Project Cedar","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"contains","field":"text","value":"no attachment","pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Could you please send the draft by Friday using the shared folder Project Cedar? No attachment is needed. Thank you!"}]}}
{"repeat":3,"prompt_id":"lookup-06","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:22:17.813Z","latency_ms":7384.994249999989,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921342-VM8kFP5TEDyys1xgMqOy"],"price_microusd":891,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30"},"grade":{"pass":true,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":true,"actual":"09:30"}]}}
{"repeat":3,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:22:19.580Z","latency_ms":7892.241625000024,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921345-zNZhUDpBCvFPTtuTsALN"],"price_microusd":2550,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:22:20.272Z","latency_ms":12204.223290999944,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"799d5366-3712-4390-aba5-30592387bed0","answer_receipt_ids":["gen-1789921351-MOzpeUos2sHiQStLUBRB"],"price_microusd":931,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":3,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:22:22.101Z","latency_ms":11339.725916999974,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.82,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"8fd2a77e-55f8-4496-aef2-067225cfdc2c","answer_receipt_ids":["gen-1789921352-uVrqbyUJiS2sVrrauMEx"],"price_microusd":106,"charged_cents":0,"answer":{"plan":"B","cost_dollars":7.8},"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":true,"actual":"B"},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":7.8}]}}
{"repeat":3,"prompt_id":"lookup-06","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":8,"started_at":"2026-09-20T16:22:25.199Z","latency_ms":10290.20633399993,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"5e44dd99-7d4e-442f-86b4-4da717552cb1","answer_receipt_ids":["gen-1789921354-LRLbRR6HAslsCrmESEC0"],"price_microusd":51,"charged_cents":0,"answer":{"day":"Thursday","time_utc":"09:30 UTC"},"grade":{"pass":false,"checks":[{"op":"eq","field":"day","value":"Thursday","pass":true,"actual":"Thursday"},{"op":"eq","field":"time_utc","value":"09:30","pass":false,"actual":"09:30 UTC"}]}}
{"repeat":3,"prompt_id":"multi-step-10","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":9,"started_at":"2026-09-20T16:22:27.473Z","latency_ms":10373.796625000017,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.79,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"4181963f-3c96-4bd2-b768-81c3fd2d6045","answer_receipt_ids":["gen-1789921355-KyR599Sr4NXgy6cFd1QR"],"price_microusd":2584,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"seconds","value":25,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-05","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":10,"started_at":"2026-09-20T16:22:32.477Z","latency_ms":7045.987500000047,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921358-189Xw9NykE2c1iuckh4a"],"price_microusd":897,"charged_cents":0,"answer":{"text":"The export tool supports CSV and JSON formats."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"CSV","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"contains","field":"text","value":"JSON","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"max_words","field":"text","value":20,"pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"revolutionary","pass":true,"actual":"The export tool supports CSV and JSON formats."},{"op":"not_contains","field":"text","value":"best","pass":true,"actual":"The export tool supports CSV and JSON formats."}]}}
{"repeat":3,"prompt_id":"multi-step-09","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":11,"started_at":"2026-09-20T16:22:33.441Z","latency_ms":8913.00524999993,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921359-Ck0AL6dH2eKvIcXFluj8"],"price_microusd":3796,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"plan","value":"B","pass":false,"actual":null},{"op":"eq","field":"cost_dollars","value":6.8,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:22:35.490Z","latency_ms":9682.218166999985,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921362-quWa535YgwSONnlY8KKw"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-02","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:22:39.523Z","latency_ms":8247.900166999898,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921366-vvKnBDzhQAy0eiZTkAMc"],"price_microusd":644,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":3,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:22:37.847Z","latency_ms":12977.361416000058,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"addb7bac-2ae2-41b5-9380-9e29bb8d43d4","answer_receipt_ids":["gen-1789921369-zH8RYOcDCmGncWw63oSi"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-09","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:22:42.355Z","latency_ms":11461.097500000033,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"2db4b4cd-4e39-4808-91da-57bccce97f89","answer_receipt_ids":["gen-1789921373-ZWdWRt7Y2cqnG8NBCvyo"],"price_microusd":44,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"multi-step-02","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":12,"started_at":"2026-09-20T16:22:45.172Z","latency_ms":9767.08683299995,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.94,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"604151dd-0ab4-474c-bc3b-deb7215714d6","answer_receipt_ids":["gen-1789921374-veVL8QurG4uPv4o10yC0"],"price_microusd":96,"charged_cents":0,"answer":{"total_dollars":66.6},"grade":{"pass":false,"checks":[{"op":"eq","field":"total_dollars","value":67.2,"pass":false,"actual":66.6}]}}
{"repeat":3,"prompt_id":"high-stakes-05","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":13,"started_at":"2026-09-20T16:22:50.824Z","latency_ms":5858.284707999905,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921375-08gPlTqxrY4Fu8Vv2hph"],"price_microusd":964,"charged_cents":0,"answer":{"decision":"REFER","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"REFER","pass":true,"actual":"REFER"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-02","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":14,"started_at":"2026-09-20T16:22:47.771Z","latency_ms":9695.065333000035,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"73b2aa48-476d-451e-a17b-e4b4a772775f","answer_receipt_ids":["gen-1789921376-WMOEvijyWZFaUlMpAI52"],"price_microusd":47,"charged_cents":0,"answer":{"days":180},"grade":{"pass":true,"checks":[{"op":"eq","field":"days","value":180,"pass":true,"actual":180}]}}
{"repeat":3,"prompt_id":"lookup-09","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":15,"started_at":"2026-09-20T16:22:53.817Z","latency_ms":5191.471084000077,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921377-SlDjSBRBEr3U5OdVyfo2"],"price_microusd":557,"charged_cents":0,"answer":{"supported":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"supported","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:22:54.940Z","latency_ms":5693.025499999989,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921379-FNfylwDJODMA4tD6l7Xj"],"price_microusd":1252,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":3,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:22:56.683Z","latency_ms":5728.423124999972,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921380-taxmNfThomKD3TyR9fsD"],"price_microusd":1339,"charged_cents":0,"answer":{"text":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Migration moved to October 7. Planned downtime: 15 minutes. Owner: Rina."}]}}
{"repeat":3,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:22:59.009Z","latency_ms":7484.37200000009,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921383-R8NrcJSvceVZO0ZcbtuD"],"price_microusd":5918,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:22:57.467Z","latency_ms":12973.031084000017,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.77,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a6c44f80-b6bd-4e08-aa3e-944a099e5d81","answer_receipt_ids":["gen-1789921388-uwSDnJN7xXMFaLrfL2X0"],"price_microusd":2153,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":3,"prompt_id":"tool-planning-07","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":16,"started_at":"2026-09-20T16:23:00.633Z","latency_ms":11943.102083000005,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.7,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"9062e204-cfac-47eb-83c1-8a6c168c744c","answer_receipt_ids":["gen-1789921391-qDsW1V7ygKXgWxJQwSvw"],"price_microusd":1288,"charged_cents":0,"answer":{"tool":"calendar.freebusy","date":"2026-10-06"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"calendar.freebusy","pass":true,"actual":"calendar.freebusy"},{"op":"eq","field":"date","value":"2026-10-06","pass":true,"actual":"2026-10-06"}]}}
{"repeat":3,"prompt_id":"rewriting-01","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":17,"started_at":"2026-09-20T16:23:02.412Z","latency_ms":12215.312208000105,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.42,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"0378003d-5a83-4a51-8320-0d5bbad2c94c","answer_receipt_ids":["gen-1789921392-rQ8rUwDxILk0K440L8rf"],"price_microusd":1451,"charged_cents":0,"answer":{"text":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"October 7","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"15 minutes","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"contains","field":"text","value":"Rina","pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The migration is moved to October 7 with a planned downtime of 15 minutes. Owner: Rina."}]}}
{"repeat":3,"prompt_id":"rewriting-07","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":18,"started_at":"2026-09-20T16:23:10.440Z","latency_ms":6942.95849999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921394-i1f6PKvsuZPZApGYBDfL"],"price_microusd":2143,"charged_cents":0,"answer":{"text":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"September 23","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"contains","field":"text","value":"AB-2049","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"max_words","field":"text","value":40,"pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"refund","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."},{"op":"not_contains","field":"text","value":"guarantee","pass":true,"actual":"We wanted to let you know that your package (tracking number: AB-2049) has been delayed, with an updated expected delivery date of September 23. Unfortunately, we're unable to offer an earlier delivery date at this time. We appreciate your patience."}]}}
{"repeat":3,"prompt_id":"multi-step-06","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":19,"started_at":"2026-09-20T16:23:06.493Z","latency_ms":14306.81320799992,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.72,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"7ceea6bf-3460-4644-908d-39a836ad8005","answer_receipt_ids":["gen-1789921397-w0nvtTUEb1TdJENCDGue"],"price_microusd":5954,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"start_utc","value":"10:00","pass":false,"actual":null},{"op":"eq","field":"end_utc","value":"10:30","pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:23:14.628Z","latency_ms":9338.953582999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921401-1IHgb7WSR3tHWByO3Sh9"],"price_microusd":2977,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-05","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:23:12.577Z","latency_ms":13466.378083000076,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"2388eb5c-5d2e-428a-9bb1-b5da603b9162","answer_receipt_ids":["gen-1789921405-mNEzXOc9hweisglXjeGW"],"price_microusd":46,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":3,"prompt_id":"lookup-10","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:23:17.383Z","latency_ms":10022.59279200004,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"16154ec5-5e75-44ac-9811-b1985d1f57be","answer_receipt_ids":["gen-1789921406-XE3j7iGzjGXLRvYJQuM0"],"price_microusd":49,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":3,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:23:20.801Z","latency_ms":8224.035416999948,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.27,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c7dbfd7d-2d6f-4715-b8ef-a941c0b782a8","answer_receipt_ids":["gen-1789921407-Vb4ewf46EfZX22fXnsYI"],"price_microusd":1123,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-05","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":20,"started_at":"2026-09-20T16:23:26.044Z","latency_ms":4530.082874999964,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921409-Ijvhiorm9kFYmoHB1OFu"],"price_microusd":639,"charged_cents":0,"answer":{"method":"POST"},"grade":{"pass":true,"checks":[{"op":"eq","field":"method","value":"POST","pass":true,"actual":"POST"}]}}
{"repeat":3,"prompt_id":"multi-step-07","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":21,"started_at":"2026-09-20T16:23:23.967Z","latency_ms":10087.313667000039,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.86,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"1e1ac2c8-2fe1-48c5-975c-216d70863000","answer_receipt_ids":["gen-1789921413-pc4svW5IAdALcLrP2M7y"],"price_microusd":82,"charged_cents":0,"answer":{"unreserved_units":54},"grade":{"pass":true,"checks":[{"op":"eq","field":"unreserved_units","value":54,"pass":true,"actual":54}]}}
{"repeat":3,"prompt_id":"lookup-10","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":22,"started_at":"2026-09-20T16:23:27.407Z","latency_ms":8289.024749999982,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921414-NoJinq6az0LZuICFg1CK"],"price_microusd":773,"charged_cents":0,"answer":{"time_utc":"17:00"},"grade":{"pass":true,"checks":[{"op":"eq","field":"time_utc","value":"17:00","pass":true,"actual":"17:00"}]}}
{"repeat":3,"prompt_id":"high-stakes-01","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":23,"started_at":"2026-09-20T16:23:29.026Z","latency_ms":11583.02812500007,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921419-2vsMc8jNjdNLFNM2DWzq"],"price_microusd":1087,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:23:30.574Z","latency_ms":11471.447000000044,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921420-l97uDnd3dwdzDcea44u9"],"price_microusd":1047,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:23:34.055Z","latency_ms":9830.051124999998,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921422-SuC143mLrd2kHFYNCjpw"],"price_microusd":1334,"charged_cents":0,"answer":{"text":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Before running sync_records, set retries to 3 and timeout_ms to 5000."}]}}
{"repeat":3,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:23:35.696Z","latency_ms":10971.047332999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921424-eIrpIeKZx2SM3UAJO6Qx"],"price_microusd":3652,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"lookup-03","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:23:40.609Z","latency_ms":10344.292415999924,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":1,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"e108bef4-1bfb-4ac4-b0d5-f457089198f7","answer_receipt_ids":["gen-1789921430-yc2waRhtvFezme78Ka5K"],"price_microusd":47,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"high-stakes-08","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":24,"started_at":"2026-09-20T16:23:42.046Z","latency_ms":10576.206790999975,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.49,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"1a3ffa01-4e34-421e-87a2-ed65ed49f5b2","answer_receipt_ids":["gen-1789921431-3cb4YKh4m4L5EUH8MH1y"],"price_microusd":1083,"charged_cents":0,"answer":{"decision":"SKIP","human_review":false},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"SKIP","pass":true,"actual":"SKIP"},{"op":"eq","field":"human_review","value":false,"pass":true,"actual":false}]}}
{"repeat":3,"prompt_id":"rewriting-10","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":25,"started_at":"2026-09-20T16:23:43.885Z","latency_ms":9937.540291000041,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.31,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"b00caeb7-f262-4022-a8ea-ce77f0b56fa9","answer_receipt_ids":["gen-1789921432-TedRMu6sSHS1YMjdOczX"],"price_microusd":115,"charged_cents":0,"answer":{"text":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"retries","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"3","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"timeout_ms","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"5000","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"contains","field":"text","value":"sync_records","pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Set retries to 3 and timeout_ms to 5000, then run sync_records."}]}}
{"repeat":3,"prompt_id":"multi-step-04","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":26,"started_at":"2026-09-20T16:23:46.668Z","latency_ms":8049.31941700005,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.8,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"c4886a57-e98f-4438-9a20-82a976e4cc90","answer_receipt_ids":["gen-1789921434-EPmxwmTlx4B3bZ0m6sES"],"price_microusd":86,"charged_cents":0,"answer":{"records_per_group":56},"grade":{"pass":false,"checks":[{"op":"eq","field":"records_per_group","value":63,"pass":false,"actual":56}]}}
{"repeat":3,"prompt_id":"lookup-03","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":27,"started_at":"2026-09-20T16:23:50.954Z","latency_ms":5746.69379099994,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921434-QJHuwyn0Avon8w0O1Bw3"],"price_microusd":732,"charged_cents":0,"answer":{"version":"v2.5"},"grade":{"pass":true,"checks":[{"op":"eq","field":"version","value":"v2.5","pass":true,"actual":"v2.5"}]}}
{"repeat":3,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:23:53.823Z","latency_ms":5325.581417000038,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921437-bI9cpjXl1HfOdCci3gJq"],"price_microusd":1339,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":3,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:23:54.718Z","latency_ms":7297.50462500006,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921440-GoTUS6upXYCtxCIed8Um"],"price_microusd":1010,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:23:52.623Z","latency_ms":11436.585374999908,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.45,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"031ff7b7-da64-442d-aa77-c91cde6c1401","answer_receipt_ids":["gen-1789921442-CIwoWuMiqD9h0OLaNwzR"],"price_microusd":1106,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":3,"prompt_id":"lookup-04","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:23:56.701Z","latency_ms":9305.102916000062,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.99,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"af772e0e-49a8-466e-b91d-88539bacff12","answer_receipt_ids":["gen-1789921445-1bb8uW9MbnPEPCy5Bu59"],"price_microusd":51,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":3,"prompt_id":"tool-planning-06","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":28,"started_at":"2026-09-20T16:24:04.060Z","latency_ms":4051.9662909999024,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921446-lBgliMHhjhcyNl8mMjpc"],"price_microusd":1072,"charged_cents":0,"answer":{"tool":"files.read","path":"/notes/launch.txt"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"files.read","pass":true,"actual":"files.read"},{"op":"eq","field":"path","value":"/notes/launch.txt","pass":true,"actual":"/notes/launch.txt"}]}}
{"repeat":3,"prompt_id":"tool-planning-04","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":29,"started_at":"2026-09-20T16:23:59.149Z","latency_ms":10748.23054099991,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.39,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"877ba899-7913-4e8f-9a07-c345b5690806","answer_receipt_ids":["gen-1789921448-DyAVVs03rDA77NlKvhIZ"],"price_microusd":1375,"charged_cents":0,"answer":{"tool":"watches.create","query":"used piano Boston","interval_minutes":60},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"watches.create","pass":true,"actual":"watches.create"},{"op":"eq","field":"query","value":"used piano Boston","pass":true,"actual":"used piano Boston"},{"op":"eq","field":"interval_minutes","value":60,"pass":true,"actual":60}]}}
{"repeat":3,"prompt_id":"high-stakes-07","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":30,"started_at":"2026-09-20T16:24:02.016Z","latency_ms":9727.517749999999,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d4055e31-131a-46b6-874f-ad83b4b186cf","answer_receipt_ids":["gen-1789921450-jC69NnHTfdSh8UzwcUBo"],"price_microusd":1046,"charged_cents":0,"answer":{"decision":"PAUSE","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"PAUSE","pass":true,"actual":"PAUSE"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"lookup-04","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":31,"started_at":"2026-09-20T16:24:06.007Z","latency_ms":7576.985042000073,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921452-iBD7NUi7rBxhFDmY8zae"],"price_microusd":861,"charged_cents":0,"answer":{"owner":"Ivo","status":"blocked"},"grade":{"pass":true,"checks":[{"op":"eq","field":"owner","value":"Ivo","pass":true,"actual":"Ivo"},{"op":"eq","field":"status","value":"blocked","pass":true,"actual":"blocked"}]}}
{"repeat":3,"prompt_id":"lookup-07","bucket":"lookup","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:24:09.898Z","latency_ms":5835.183000000077,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921454-azBYgINAlAH6DqzTJXVw"],"price_microusd":773,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":3,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:24:13.585Z","latency_ms":4504.560540999984,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921456-SLGTG7axJDuipEeBhKzF"],"price_microusd":1061,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:24:08.113Z","latency_ms":11716.910582999932,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.33,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"70d75f06-81e3-4d13-b77a-a561134cd832","answer_receipt_ids":["gen-1789921458-xm6u4j3sqOT8nfMAoqHc"],"price_microusd":1587,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":3,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:24:11.744Z","latency_ms":10962.326207999955,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.8,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"b355725d-0cce-4bbe-bb30-ccbeed306af0","answer_receipt_ids":["gen-1789921461-LsgypZjxnvdbb2sz2NlC"],"price_microusd":92,"charged_cents":0,"answer":{"finish_minute":12},"grade":{"pass":true,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":true,"actual":12}]}}
{"repeat":3,"prompt_id":"tool-planning-10","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":32,"started_at":"2026-09-20T16:24:19.830Z","latency_ms":5530.842958000023,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921463-2SrfBpB9vRyX1mqXvZ7J"],"price_microusd":1551,"charged_cents":0,"answer":{"tool":"github.search_issues","repo":"acme/mobile","query":"NullPointerException upload"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"github.search_issues","pass":true,"actual":"github.search_issues"},{"op":"eq","field":"repo","value":"acme/mobile","pass":true,"actual":"acme/mobile"},{"op":"eq","field":"query","value":"NullPointerException upload","pass":true,"actual":"NullPointerException upload"}]}}
{"repeat":3,"prompt_id":"lookup-07","bucket":"lookup","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":33,"started_at":"2026-09-20T16:24:15.734Z","latency_ms":10322.442541999975,"decision":{"lane":"fast","recommended":"fast","reason":[],"confidence":0.9,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"fast","final_lane":"fast","fallback":false,"router_transaction_id":"fc3384e1-0f15-42be-a136-038e4a7102aa","answer_receipt_ids":["gen-1789921465-X9pkt1TkkcholbVyUpDC"],"price_microusd":49,"charged_cents":0,"answer":{"plan":"studio","day":6},"grade":{"pass":true,"checks":[{"op":"eq","field":"plan","value":"studio","pass":true,"actual":"studio"},{"op":"eq","field":"day","value":6,"pass":true,"actual":6}]}}
{"repeat":3,"prompt_id":"multi-step-03","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":34,"started_at":"2026-09-20T16:24:22.707Z","latency_ms":6511.251083999989,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921466-Su97MdXpPiWwbiJ2aZb3"],"price_microusd":4383,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"finish_minute","value":12,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"high-stakes-02","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":35,"started_at":"2026-09-20T16:24:18.090Z","latency_ms":14080.530374999973,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.45,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"c6cae715-aaac-42dc-95d3-698eb2f173f4","answer_receipt_ids":["gen-1789921469-bAhDzOqJSH0IoWU5chrf"],"price_microusd":1097,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:24:25.362Z","latency_ms":9786.08050000004,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921473-TQ6c8dEYJKc4zvjPreq0"],"price_microusd":1020,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:24:26.057Z","latency_ms":12047.951957999961,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921475-obkATWMxekJf7CfB3mQM"],"price_microusd":3518,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"numerator","value":7,"pass":false,"actual":null},{"op":"eq","field":"denominator","value":13,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:24:29.219Z","latency_ms":13782.846542000072,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.71,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8ae84590-a9bd-4091-8f1f-1380b9ad4205","answer_receipt_ids":["gen-1789921481-F59z71ZfrFFIgip0qo6h"],"price_microusd":1183,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":3,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:24:32.171Z","latency_ms":12447.638750000042,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.45,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"bf9f7358-d5f4-443b-ba00-0b51b0d245c6","answer_receipt_ids":["gen-1789921483-FtkE9Ej34DRkj88EjVot"],"price_microusd":1173,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":3,"prompt_id":"high-stakes-04","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":36,"started_at":"2026-09-20T16:24:35.148Z","latency_ms":11205.061541000032,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.25,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"8da1e0f3-b883-473f-b767-538fd8550d2a","answer_receipt_ids":["gen-1789921484-RVjbraD4LjPNbmWnAJeL"],"price_microusd":1056,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-05","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":37,"started_at":"2026-09-20T16:24:38.105Z","latency_ms":9334.780207999982,"decision":{"lane":"balanced","recommended":"balanced","reason":["reasoning_tools_or_verification_floor"],"confidence":0.89,"policy_escalated":false,"risk_bump_applied":false,"high_stakes":false,"needs_tools":false,"needs_verification":true},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"27bfe9e4-f756-424a-95f7-0814fa03b939","answer_receipt_ids":["gen-1789921486-lMaIBnoFPb4tzEJaZ0R6"],"price_microusd":105,"charged_cents":0,"answer":{"numerator":7,"denominator":13},"grade":{"pass":true,"checks":[{"op":"eq","field":"numerator","value":7,"pass":true,"actual":7},{"op":"eq","field":"denominator","value":13,"pass":true,"actual":13}]}}
{"repeat":3,"prompt_id":"tool-planning-03","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":38,"started_at":"2026-09-20T16:24:43.002Z","latency_ms":6199.116792000015,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921487-qtXDvSCfTkIYkw7Gg1oQ"],"price_microusd":1149,"charged_cents":0,"answer":{"tool":"mail.search","query":"from:lumen invoice"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"mail.search","pass":true,"actual":"mail.search"},{"op":"eq","field":"query","value":"from:lumen invoice","pass":true,"actual":"from:lumen invoice"}]}}
{"repeat":3,"prompt_id":"tool-planning-05","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":39,"started_at":"2026-09-20T16:24:44.619Z","latency_ms":6316.198709000018,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921489-4L3ogkrs49Nod5pVntAQ"],"price_microusd":1139,"charged_cents":0,"answer":{"tool":"jobs.result","job_id":"job-73"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"jobs.result","pass":true,"actual":"jobs.result"},{"op":"eq","field":"job_id","value":"job-73","pass":true,"actual":"job-73"}]}}
{"repeat":3,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:24:46.354Z","latency_ms":7699.802582999924,"decision":{"lane":"balanced","recommended":"fast","reason":["low_confidence"],"confidence":0.31,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"balanced","final_lane":"balanced","fallback":false,"router_transaction_id":"da16d37b-3261-4bbc-9a4e-b21856157df7","answer_receipt_ids":["gen-1789921493-adtmQIRDQqDfvqZhsEJ9"],"price_microusd":117,"charged_cents":0,"answer":{"text":"The password reset link lasts for 20 minutes. If it expires, please ask for a new link."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"The password reset link lasts for 20 minutes. If it expires, please ask for a new link."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"The password reset link lasts for 20 minutes. If it expires, please ask for a new link."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"The password reset link lasts for 20 minutes. If it expires, please ask for a new link."}]}}
{"repeat":3,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:24:47.440Z","latency_ms":8536.682833999977,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.28,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"3ea8a5ca-ebaf-4bd7-9e8d-4d455cda3981","answer_receipt_ids":["gen-1789921494-Fkpw3o2Z1nD19tYg1Mmf"],"price_microusd":1240,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":3,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:24:49.201Z","latency_ms":8996.159375000047,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.55,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"a93ab1b0-a5b3-41f6-b54c-866886a90ba6","answer_receipt_ids":["gen-1789921496-pr9AWLnGjTixx6Aj2mRN"],"price_microusd":1415,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:24:50.936Z","latency_ms":8932.179332999978,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.63,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"d787f512-7ad9-4830-b153-4e95e674849b","answer_receipt_ids":["gen-1789921498-fERxdm2hedphgs8jHq4H"],"price_microusd":998,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"rewriting-03","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":40,"started_at":"2026-09-20T16:24:54.054Z","latency_ms":10013.449292000034,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921501-FRJ4W9U5WE3Lbfu86V7o"],"price_microusd":1370,"charged_cents":0,"answer":{"text":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"20 minutes","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"contains","field":"text","value":"new link","pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."},{"op":"max_words","field":"text","value":30,"pass":true,"actual":"Your password reset link is only valid for 20 minutes. If it expires, request a new link to reset your password."}]}}
{"repeat":3,"prompt_id":"rewriting-06","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":41,"started_at":"2026-09-20T16:24:55.977Z","latency_ms":10285.787249999936,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921504-M4uOweWCefJKugz3utKH"],"price_microusd":1206,"charged_cents":0,"answer":{"text":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"config.yaml","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"contains","field":"text","value":"rerun","pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Build failed: config.yaml is missing. Restore config.yaml and rerun the build."}]}}
{"repeat":3,"prompt_id":"rewriting-04","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":42,"started_at":"2026-09-20T16:24:58.198Z","latency_ms":10069.492667000042,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921506-xZM2uw3VreD5JLe2R8mU"],"price_microusd":1381,"charged_cents":0,"answer":{"text":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"09:00","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"Monday","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"contains","field":"text","value":"18 Pine Street","pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"},{"op":"max_words","field":"text","value":25,"pass":true,"actual":"Our studio opens at 09:00 on Monday at 18 Pine Street. We look forward to welcoming you!"}]}}
{"repeat":3,"prompt_id":"high-stakes-06","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":43,"started_at":"2026-09-20T16:24:59.869Z","latency_ms":10120.510624999995,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921508-wD3O7iPnl2IS7IEciDeZ"],"price_microusd":964,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:25:04.068Z","latency_ms":8499.624583999976,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921510-knPgZARuAUhZmUBOGIwY"],"price_microusd":2344,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:25:08.268Z","latency_ms":7503.063291999977,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921513-vUNHB1gmxJbrxPYdOb2L"],"price_microusd":1813,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":3,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:25:06.264Z","latency_ms":12896.952082999982,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"611346d1-926a-4441-8bb5-90305606c27f","answer_receipt_ids":["gen-1789921517-LnqazzWKXRyvmDKHpYxe"],"price_microusd":1157,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":3,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:25:09.990Z","latency_ms":13921.987333000056,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.56,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"35005b29-6db8-4469-bfd9-1e858a497459","answer_receipt_ids":["gen-1789921520-NOa5PIBBolj0KszhnZpX"],"price_microusd":4152,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"multi-step-01","bucket":"multi-step","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":44,"started_at":"2026-09-20T16:25:12.569Z","latency_ms":13912.533708999981,"decision":{"lane":"frontier","recommended":"balanced","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"dff55918-7260-45d7-b434-8ddf350bb327","answer_receipt_ids":["gen-1789921524-VYGQEFaBu3yNxHAwxDLz"],"price_microusd":2145,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"minutes","value":5,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-09","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":45,"started_at":"2026-09-20T16:25:19.162Z","latency_ms":8956.366750000045,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921526-8yWfka642xTulVBbHRCF"],"price_microusd":1123,"charged_cents":0,"answer":{"tool":"video.validate","composition_id":"comp-21"},"grade":{"pass":true,"checks":[{"op":"eq","field":"tool","value":"video.validate","pass":true,"actual":"video.validate"},{"op":"eq","field":"composition_id","value":"comp-21","pass":true,"actual":"comp-21"}]}}
{"repeat":3,"prompt_id":"rewriting-09","bucket":"rewriting","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":46,"started_at":"2026-09-20T16:25:15.772Z","latency_ms":14691.054833999951,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","low_confidence"],"confidence":0.58,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":false,"needs_tools":false,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"b6ef1717-a50c-40f4-bda9-a0ced696997f","answer_receipt_ids":["gen-1789921528-kRSg8cXy2nLgNVM9grir"],"price_microusd":1847,"charged_cents":0,"answer":{"text":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},"grade":{"pass":true,"checks":[{"op":"contains","field":"text","value":"Search","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"Saved documents","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"contains","field":"text","value":"16:30 UTC","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"max_words","field":"text","value":35,"pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."},{"op":"not_contains","field":"text","value":"resolved","pass":true,"actual":"Search is currently unavailable. Saved documents are unaffected. We are working on restoring functionality and will provide the next update at 16:30 UTC. Thank you for your patience."}]}}
{"repeat":3,"prompt_id":"high-stakes-10","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":47,"started_at":"2026-09-20T16:25:23.913Z","latency_ms":10543.353082999936,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921530-Ubzqk70mf2LuoLyICnjA"],"price_microusd":4116,"charged_cents":0,"answer":null,"grade":{"pass":false,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":false,"actual":null},{"op":"eq","field":"human_review","value":true,"pass":false,"actual":null}]}}
{"repeat":3,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:25:26.482Z","latency_ms":9591.54483300005,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921534-7RFxCdeU0u8gRDowzarG"],"price_microusd":891,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":3,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"control","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:25:28.119Z","latency_ms":9621.381999999983,"decision":null,"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":null,"answer_receipt_ids":["gen-1789921536-z6hOLcEnpYOENj2Xg6OR"],"price_microusd":948,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}
{"repeat":3,"prompt_id":"tool-planning-08","bucket":"tool-planning","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":48,"started_at":"2026-09-20T16:25:36.074Z","latency_ms":4490.39383299998,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes"],"confidence":0.82,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":true,"needs_verification":false},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"62a0cf2b-843d-4205-a461-96f4c8d5e0f3","answer_receipt_ids":["gen-1789921539-qrKE77MG6MgFkYf3zQqV"],"price_microusd":925,"charged_cents":0,"answer":{"tool":"wallet.balance()"},"grade":{"pass":false,"checks":[{"op":"eq","field":"tool","value":"wallet.balance","pass":false,"actual":"wallet.balance()"}]}}
{"repeat":3,"prompt_id":"high-stakes-03","bucket":"high-stakes","arm":"treatment","run_id":"jev-20260920-50x3-metered","sequence":49,"started_at":"2026-09-20T16:25:37.741Z","latency_ms":4521.456875000033,"decision":{"lane":"frontier","recommended":"fast","reason":["reasoning_tools_or_verification_floor","high_stakes","low_confidence"],"confidence":0.52,"policy_escalated":true,"risk_bump_applied":true,"high_stakes":true,"needs_tools":false,"needs_verification":true},"first_lane":"frontier","final_lane":"frontier","fallback":false,"router_transaction_id":"f070f928-b4ee-41af-ba1a-4802fac92611","answer_receipt_ids":["gen-1789921540-BVm95iZ717meuQJzV9Dw"],"price_microusd":982,"charged_cents":0,"answer":{"decision":"HOLD","human_review":true},"grade":{"pass":true,"checks":[{"op":"eq","field":"decision","value":"HOLD","pass":true,"actual":"HOLD"},{"op":"eq","field":"human_review","value":true,"pass":true,"actual":true}]}}