{
  "run_id": "jev-20260920-50x3-metered",
  "completed_at": "2026-09-20T16:25:42.404Z",
  "accounting": "Sum actual per-call customer prices returned by Vaaya: Jev data.vaaya_billing.price_microusd; non-streaming LLM API usage.cost converted to micro-USD (already includes Vaaya fee). Count each call once, including fallbacks and charged failures. Do not add a markup again or count aggregate settlement twice. No reconstructed prices.",
  "control": {
    "arm": "control",
    "repeat": null,
    "answers": 150,
    "passing": 113,
    "pass_rate_percent": 75.33333333333333,
    "calls": 150,
    "price_microusd": 257437,
    "price_usd": 0.257437,
    "cost_per_passing_answer_usd": 0.002278203539823009,
    "router_price_usd": 0,
    "router_provider_usage_usd": 0,
    "metered_upstream_cost_usd": 0.249866,
    "p95_latency_seconds": 11.471447000000044,
    "policy_promotions": 0,
    "policy_promotion_rate_percent": 0,
    "fallback_answers": 0,
    "fallback_rate_percent": 0,
    "malformed_final_answers": 34,
    "failed_calls": 1,
    "lanes": {
      "fast": 0,
      "balanced": 0,
      "frontier": 150
    },
    "buckets": {
      "lookup": {
        "passing": 30,
        "total": 30
      },
      "rewriting": {
        "passing": 30,
        "total": 30
      },
      "tool-planning": {
        "passing": 27,
        "total": 30
      },
      "multi-step": {
        "passing": 0,
        "total": 30
      },
      "high-stakes": {
        "passing": 26,
        "total": 30
      }
    }
  },
  "treatment": {
    "arm": "treatment",
    "repeat": null,
    "answers": 150,
    "passing": 120,
    "pass_rate_percent": 80,
    "calls": 300,
    "price_microusd": 168121,
    "price_usd": 0.168121,
    "cost_per_passing_answer_usd": 0.0014010083333333332,
    "router_price_usd": 0.005151,
    "router_provider_usage_usd": 0.004885902000000003,
    "metered_upstream_cost_usd": 0.163106,
    "p95_latency_seconds": 15.359868791999993,
    "policy_promotions": 101,
    "policy_promotion_rate_percent": 67.33333333333333,
    "fallback_answers": 0,
    "fallback_rate_percent": 0,
    "malformed_final_answers": 14,
    "failed_calls": 0,
    "lanes": {
      "fast": 30,
      "balanced": 22,
      "frontier": 98
    },
    "buckets": {
      "lookup": {
        "passing": 27,
        "total": 30
      },
      "rewriting": {
        "passing": 30,
        "total": 30
      },
      "tool-planning": {
        "passing": 27,
        "total": 30
      },
      "multi-step": {
        "passing": 9,
        "total": 30
      },
      "high-stakes": {
        "passing": 27,
        "total": 30
      }
    }
  },
  "repeats": [
    {
      "repeat": 1,
      "control": {
        "arm": "control",
        "repeat": 1,
        "answers": 50,
        "passing": 37,
        "pass_rate_percent": 74,
        "calls": 50,
        "price_microusd": 85290,
        "price_usd": 0.08529,
        "cost_per_passing_answer_usd": 0.002305135135135135,
        "router_price_usd": 0,
        "router_provider_usage_usd": 0,
        "metered_upstream_cost_usd": 0.082782,
        "p95_latency_seconds": 11.983670125000005,
        "policy_promotions": 0,
        "policy_promotion_rate_percent": 0,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 12,
        "failed_calls": 1,
        "lanes": {
          "fast": 0,
          "balanced": 0,
          "frontier": 50
        },
        "buckets": {
          "lookup": {
            "passing": 10,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 0,
            "total": 10
          },
          "high-stakes": {
            "passing": 8,
            "total": 10
          }
        }
      },
      "treatment": {
        "arm": "treatment",
        "repeat": 1,
        "answers": 50,
        "passing": 40,
        "pass_rate_percent": 80,
        "calls": 100,
        "price_microusd": 56766,
        "price_usd": 0.056766,
        "cost_per_passing_answer_usd": 0.0014191499999999999,
        "router_price_usd": 0.001717,
        "router_provider_usage_usd": 0.0016286339999999995,
        "metered_upstream_cost_usd": 0.055073,
        "p95_latency_seconds": 15.598847917000006,
        "policy_promotions": 34,
        "policy_promotion_rate_percent": 68,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 5,
        "failed_calls": 0,
        "lanes": {
          "fast": 10,
          "balanced": 7,
          "frontier": 33
        },
        "buckets": {
          "lookup": {
            "passing": 9,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 3,
            "total": 10
          },
          "high-stakes": {
            "passing": 9,
            "total": 10
          }
        }
      }
    },
    {
      "repeat": 2,
      "control": {
        "arm": "control",
        "repeat": 2,
        "answers": 50,
        "passing": 38,
        "pass_rate_percent": 76,
        "calls": 50,
        "price_microusd": 85971,
        "price_usd": 0.085971,
        "cost_per_passing_answer_usd": 0.0022623947368421056,
        "router_price_usd": 0,
        "router_provider_usage_usd": 0,
        "metered_upstream_cost_usd": 0.083442,
        "p95_latency_seconds": 11.379240417000023,
        "policy_promotions": 0,
        "policy_promotion_rate_percent": 0,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 11,
        "failed_calls": 0,
        "lanes": {
          "fast": 0,
          "balanced": 0,
          "frontier": 50
        },
        "buckets": {
          "lookup": {
            "passing": 10,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 0,
            "total": 10
          },
          "high-stakes": {
            "passing": 9,
            "total": 10
          }
        }
      },
      "treatment": {
        "arm": "treatment",
        "repeat": 2,
        "answers": 50,
        "passing": 40,
        "pass_rate_percent": 80,
        "calls": 100,
        "price_microusd": 55670,
        "price_usd": 0.05567,
        "cost_per_passing_answer_usd": 0.00139175,
        "router_price_usd": 0.001717,
        "router_provider_usage_usd": 0.0016286339999999995,
        "metered_upstream_cost_usd": 0.05401,
        "p95_latency_seconds": 15.459498916000012,
        "policy_promotions": 33,
        "policy_promotion_rate_percent": 66,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 4,
        "failed_calls": 0,
        "lanes": {
          "fast": 10,
          "balanced": 7,
          "frontier": 33
        },
        "buckets": {
          "lookup": {
            "passing": 9,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 3,
            "total": 10
          },
          "high-stakes": {
            "passing": 9,
            "total": 10
          }
        }
      }
    },
    {
      "repeat": 3,
      "control": {
        "arm": "control",
        "repeat": 3,
        "answers": 50,
        "passing": 38,
        "pass_rate_percent": 76,
        "calls": 50,
        "price_microusd": 86176,
        "price_usd": 0.086176,
        "cost_per_passing_answer_usd": 0.0022677894736842104,
        "router_price_usd": 0,
        "router_provider_usage_usd": 0,
        "metered_upstream_cost_usd": 0.083642,
        "p95_latency_seconds": 11.471447000000044,
        "policy_promotions": 0,
        "policy_promotion_rate_percent": 0,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 11,
        "failed_calls": 0,
        "lanes": {
          "fast": 0,
          "balanced": 0,
          "frontier": 50
        },
        "buckets": {
          "lookup": {
            "passing": 10,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 0,
            "total": 10
          },
          "high-stakes": {
            "passing": 9,
            "total": 10
          }
        }
      },
      "treatment": {
        "arm": "treatment",
        "repeat": 3,
        "answers": 50,
        "passing": 40,
        "pass_rate_percent": 80,
        "calls": 100,
        "price_microusd": 55685,
        "price_usd": 0.055685,
        "cost_per_passing_answer_usd": 0.001392125,
        "router_price_usd": 0.001717,
        "router_provider_usage_usd": 0.0016286339999999997,
        "metered_upstream_cost_usd": 0.054023,
        "p95_latency_seconds": 14.691054833999951,
        "policy_promotions": 34,
        "policy_promotion_rate_percent": 68,
        "fallback_answers": 0,
        "fallback_rate_percent": 0,
        "malformed_final_answers": 5,
        "failed_calls": 0,
        "lanes": {
          "fast": 10,
          "balanced": 8,
          "frontier": 32
        },
        "buckets": {
          "lookup": {
            "passing": 9,
            "total": 10
          },
          "rewriting": {
            "passing": 10,
            "total": 10
          },
          "tool-planning": {
            "passing": 9,
            "total": 10
          },
          "multi-step": {
            "passing": 3,
            "total": 10
          },
          "high-stakes": {
            "passing": 9,
            "total": 10
          }
        }
      }
    }
  ],
  "delta": {
    "price_usd": -0.08931600000000003,
    "pass_rate_percentage_points": 4.666666666666671,
    "p95_latency_seconds": 3.8884217919999493
  },
  "customer_price_savings_percent": 34.69431356021086,
  "p95_latency_change_percent": 33.89652405664198,
  "acceptance_rule_met": true,
  "ledger_verification": {
    "matched_usage_rows": 449,
    "uncharged_failed_requests": 1,
    "total_calls": 450,
    "all_successful_calls_matched": true
  }
}