{ "measurement_dates": ["2026-10-13", "2026-10-14"], "measurement_timezone": "America/Los_Angeles", "pricing_date": "2026-10-13", "models": { "flash35": "gemini-3.5-flash", "flash37": "gemini-3.7-flash", "baseline": "gemini-3.8-flash", "argon": "gemini-4-argon" }, "thinking_levels": { "flash35": ["low", "medium", "high"], "flash37": ["low", "medium", "high"], "baseline": ["low", "medium", "high"], "argon": ["low", "medium", "high"] }, "unique_tasks": 36, "completed_observations": 432, "new_observations": 288, "reused_observations": 144, "completed_observations_per_model_per_thinking_per_task": 1, "attempts": { "new_attempts": 291, "completed_new_attempts": 288, "first_attempt_observations": 432, "first_attempt_execution_errors": 3, "technical_retry_attempts": 3, "incomplete_attempts": [ { "task": "task-23", "configuration": "flash37_medium", "elapsed_seconds": 215.86590750142932, "elapsed_basis": "CLI wall time to error artifact", "execution_errors": 1, "known_usage_gross_usd": 0.1167557, "unknown_usage_reserve_usd": 2.9417472, "provider_timeout_seconds": 120.01234263926744 }, { "task": "task-31", "configuration": "flash35_high", "elapsed_seconds": 148.13, "elapsed_basis": "Canonical evaluation duration; CLI wall time unavailable after runner budget stop", "execution_errors": 1, "known_usage_gross_usd": 0.00814505, "unknown_usage_reserve_usd": 2.9638656, "provider_timeout_seconds": 119.95276488363743 }, { "task": "task-35", "configuration": "flash37_medium", "elapsed_seconds": 246.917, "elapsed_basis": "Canonical evaluation duration; CLI wall time unavailable after runner budget stop", "execution_errors": 1, "known_usage_gross_usd": 0.19166695, "unknown_usage_reserve_usd": 2.9417472, "provider_timeout_seconds": 119.97964370064437 } ], "configurations": { "argon_high": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "argon_low": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 32, "completed_quality_denominator": 36 }, "argon_medium": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 34, "completed_quality_denominator": 36 }, "baseline_high": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "baseline_low": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "baseline_medium": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 32, "completed_quality_denominator": 36 }, "flash35_high": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 1, "first_attempt_passed": 27, "completed_quality_denominator": 36 }, "flash35_low": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "flash35_medium": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "flash37_high": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 30, "completed_quality_denominator": 36 }, "flash37_low": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 0, "first_attempt_passed": 29, "completed_quality_denominator": 36 }, "flash37_medium": { "first_attempt_tasks": 36, "first_attempt_execution_errors": 2, "first_attempt_passed": 30, "completed_quality_denominator": 36 } } }, "quality_basis": "Pass rates conditional on completion; two separately labeled same-setting technical retries for 3.7 Medium and one for 3.5 High. All three original execution errors retained in first-attempt reliability.", "comparison_cost_basis": "Completed tasks, including quality failures and fixed helpers; excludes grading, all three original incomplete timeout attempts and unknown-usage reserves.", "cost_basis": "Provider usage metadata repriced at gross public Standard rates on 2026-10-13; promotional credits excluded. Not a billing invoice.", "latency_basis": "Customer-agent turn durations including fixed helpers and tools, excluding grading and scenario setup. Task latency sums those turn durations.", "family_basis": "Overlapping memberships; judge means are full-task scores, not independent dimension scores.", "output_token_cap_per_request": 8192, "budget": { "new_gross_debit_usd": 91.623511015, "new_known_usage_gross_usd": 82.776151015, "unknown_usage_reserve_usd": 8.84736, "previous_project_spend_usd": 40.325677755, "approved_new_cap_usd": 250, "unknown_usage_rows": 3, "new_native_trace_omitted_helper_requests": 3, "new_native_trace_omitted_helper_total_tokens": 3001, "max_tokens_finishes": 3, "combined_project_debit_usd": 131.94918877 }, "prior_unmatched_work": { "paid_fixture_mismatch_gross_usd": 0.0332036, "pre_model_setup_errors": 1, "setup_error_gross_usd": 0, "successful_smoke_gross_usd": 0.001925, "regional_error_reserve_usd": 0.02, "superseded_classic35_pilot_gross_usd": 0.527623, "superseded_classic35_completed_observations": 1, "superseded_classic35_owner_interrupted_observations": 1 }, "configurations": { "flash35_low": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.638888888888889, "agent_gross_total": 4.724264345, "agent_gross_per_task": 0.1312295651388889, "judge_gross_total": 2.098623, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 62.8735, "turn_p95_seconds": 195.7419, "task_p50_seconds": 65.4945, "task_p95_seconds": 295.971, "tokens_per_task": 205117.41666666666, "tokens_per_turn": 153838.0625, "gross_cost_per_turn": 0.09842217385416667, "main_calls": 144, "main_call_p50_seconds": 11.182500000000001, "tool_calls_per_task": 5.666666666666667, "timed_main_calls": 144, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.583333333333333, "agent_gross_total": 5.972176745, "agent_gross_per_task": 0.16589379847222221, "judge_gross_total": 2.7194385, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 42.696, "turn_p95_seconds": 133.381, "task_p50_seconds": 55.1405, "task_p95_seconds": 184.32625000000002, "tokens_per_task": 240589.38888888888, "tokens_per_turn": 180442.04166666666, "gross_cost_per_turn": 0.12442034885416665, "main_calls": 160, "main_call_p50_seconds": 6.707000000000001, "tool_calls_per_task": 7.166666666666667, "timed_main_calls": 160, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 36, "passed": 28, "failed": 8, "errors": 0, "pass_rate": 77.77777777777777, "judge_mean": 4.555555555555555, "agent_gross_total": 6.507431335, "agent_gross_per_task": 0.18076198152777778, "judge_gross_total": 2.6132865, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 90.28450000000001, "turn_p95_seconds": 206.80984999999998, "task_p50_seconds": 129.017, "task_p95_seconds": 271.92825, "tokens_per_task": 243730.36111111112, "tokens_per_turn": 182797.77083333334, "gross_cost_per_turn": 0.13557148614583334, "main_calls": 166, "main_call_p50_seconds": 18.769, "tool_calls_per_task": 7.361111111111111, "timed_main_calls": 166, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 36, "passed": 29, "failed": 7, "errors": 0, "pass_rate": 80.55555555555556, "judge_mean": 4.694444444444445, "agent_gross_total": 4.492563905, "agent_gross_per_task": 0.12479344180555556, "judge_gross_total": 2.2951425, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 30.414, "turn_p95_seconds": 109.72779999999997, "task_p50_seconds": 40.463499999999996, "task_p95_seconds": 119.6505, "tokens_per_task": 205313.13888888888, "tokens_per_turn": 153984.85416666666, "gross_cost_per_turn": 0.09359508135416666, "main_calls": 131, "main_call_p50_seconds": 4.257, "tool_calls_per_task": 5.916666666666667, "timed_main_calls": 131, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 36, "passed": 32, "failed": 4, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 4.75, "agent_gross_total": 4.16657145, "agent_gross_per_task": 0.11573809583333333, "judge_gross_total": 2.0524635, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 35.5315, "turn_p95_seconds": 88.34804999999999, "task_p50_seconds": 44.4885, "task_p95_seconds": 139.3815, "tokens_per_task": 185120.63888888888, "tokens_per_turn": 138840.47916666666, "gross_cost_per_turn": 0.086803571875, "main_calls": 119, "main_call_p50_seconds": 6.617, "tool_calls_per_task": 5.472222222222222, "timed_main_calls": 119, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.777777777777778, "agent_gross_total": 5.384371945, "agent_gross_per_task": 0.1495658873611111, "judge_gross_total": 2.424123, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 35.84, "turn_p95_seconds": 99.50089999999999, "task_p50_seconds": 43.56, "task_p95_seconds": 146.65975, "tokens_per_task": 235415.13888888888, "tokens_per_turn": 176561.35416666666, "gross_cost_per_turn": 0.11217441552083333, "main_calls": 150, "main_call_p50_seconds": 6.1625, "tool_calls_per_task": 8.055555555555555, "timed_main_calls": 150, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.694444444444445, "agent_gross_total": 4.42582472, "agent_gross_per_task": 0.12293957555555554, "judge_gross_total": 2.4816615, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 28.323999999999998, "turn_p95_seconds": 81.49584999999999, "task_p50_seconds": 32.888999999999996, "task_p95_seconds": 95.54025, "tokens_per_task": 214473.11111111112, "tokens_per_turn": 160854.83333333334, "gross_cost_per_turn": 0.09220468166666666, "main_calls": 141, "main_call_p50_seconds": 3.469, "tool_calls_per_task": 6.333333333333333, "timed_main_calls": 141, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 36, "passed": 32, "failed": 4, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 4.944444444444445, "agent_gross_total": 4.973088365, "agent_gross_per_task": 0.1381413434722222, "judge_gross_total": 2.50476, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 36.7195, "turn_p95_seconds": 93.49484999999999, "task_p50_seconds": 44.972, "task_p95_seconds": 113.0645, "tokens_per_task": 218301.55555555556, "tokens_per_turn": 163726.16666666666, "gross_cost_per_turn": 0.10360600760416666, "main_calls": 142, "main_call_p50_seconds": 5.298, "tool_calls_per_task": 6.666666666666667, "timed_main_calls": 142, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.638888888888889, "agent_gross_total": 6.815416405, "agent_gross_per_task": 0.1893171223611111, "judge_gross_total": 2.6902125, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 65.406, "turn_p95_seconds": 146.74979999999996, "task_p50_seconds": 80.027, "task_p95_seconds": 199.50050000000002, "tokens_per_task": 261113.05555555556, "tokens_per_turn": 195834.79166666666, "gross_cost_per_turn": 0.14198784177083332, "main_calls": 178, "main_call_p50_seconds": 9.339500000000001, "tool_calls_per_task": 8.472222222222221, "timed_main_calls": 178, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 36, "passed": 32, "failed": 4, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 4.861111111111111, "agent_gross_total": 15.185892805, "agent_gross_per_task": 0.42183035569444444, "judge_gross_total": 5.115276, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 29.9025, "turn_p95_seconds": 188.85654999999994, "task_p50_seconds": 36.965, "task_p95_seconds": 281.46675, "tokens_per_task": 473618.44444444444, "tokens_per_turn": 355213.8333333333, "gross_cost_per_turn": 0.3163727667708333, "main_calls": 230, "main_call_p50_seconds": 3.907, "tool_calls_per_task": 13.36111111111111, "timed_main_calls": 230, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 36, "passed": 34, "failed": 2, "errors": 0, "pass_rate": 94.44444444444444, "judge_mean": 5.0, "agent_gross_total": 12.6433971, "agent_gross_per_task": 0.351205475, "judge_gross_total": 3.1215705000000002, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 28.881, "turn_p95_seconds": 90.95604999999998, "task_p50_seconds": 36.325, "task_p95_seconds": 98.39724999999999, "tokens_per_task": 257882.0, "tokens_per_turn": 193411.5, "gross_cost_per_turn": 0.26340410625, "main_calls": 149, "main_call_p50_seconds": 3.873, "tool_calls_per_task": 8.25, "timed_main_calls": 149, "native_main_timing_omitted_requests": 0 }, "argon_high": { "observations": 36, "passed": 30, "failed": 6, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.75, "agent_gross_total": 14.38369385, "agent_gross_per_task": 0.3995470513888889, "judge_gross_total": 2.411259, "simulator_gross_total": 0, "turn_count": 48, "turn_p50_seconds": 55.153, "turn_p95_seconds": 140.6477, "task_p50_seconds": 73.8475, "task_p95_seconds": 208.55375, "tokens_per_task": 251396.69444444444, "tokens_per_turn": 188547.52083333334, "gross_cost_per_turn": 0.29966028854166665, "main_calls": 160, "main_call_p50_seconds": 7.598, "tool_calls_per_task": 8.805555555555555, "timed_main_calls": 157, "native_main_timing_omitted_requests": 3 } }, "families": { "catalog grounding": { "argon_high": { "observations": 15, "passed": 14, "failed": 1, "errors": 0, "pass_rate": 93.33333333333333, "judge_mean": 4.866666666666666, "agent_gross_total": 6.435632375, "agent_gross_per_task": 0.4290421583333333, "judge_gross_total": 1.4718435, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 78.868, "turn_p95_seconds": 156.3892, "task_p50_seconds": 83.932, "task_p95_seconds": 182.3696999999999, "tokens_per_task": 271348.93333333335, "tokens_per_turn": 239425.5294117647, "gross_cost_per_turn": 0.37856661029411764, "main_calls": 64, "main_call_p50_seconds": 7.794499999999999, "tool_calls_per_task": 12.2, "timed_main_calls": 62, "native_main_timing_omitted_requests": 2 }, "argon_low": { "observations": 15, "passed": 14, "failed": 1, "errors": 0, "pass_rate": 93.33333333333333, "judge_mean": 4.866666666666666, "agent_gross_total": 9.167582535, "agent_gross_per_task": 0.6111721689999999, "judge_gross_total": 4.299978, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 37.004, "turn_p95_seconds": 291.6175999999999, "task_p50_seconds": 46.868, "task_p95_seconds": 299.71739999999994, "tokens_per_task": 783575.3333333334, "tokens_per_turn": 691390.0, "gross_cost_per_turn": 0.5392695608823529, "main_calls": 116, "main_call_p50_seconds": 3.995, "tool_calls_per_task": 22.466666666666665, "timed_main_calls": 116, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 15, "passed": 15, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 5.957353595, "agent_gross_per_task": 0.39715690633333334, "judge_gross_total": 2.165022, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 35.938, "turn_p95_seconds": 92.70459999999996, "task_p50_seconds": 41.464, "task_p95_seconds": 99.15489999999993, "tokens_per_task": 298530.4, "tokens_per_turn": 263409.17647058825, "gross_cost_per_turn": 0.3504325644117647, "main_calls": 56, "main_call_p50_seconds": 4.3934999999999995, "tool_calls_per_task": 10.733333333333333, "timed_main_calls": 56, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 15, "passed": 14, "failed": 1, "errors": 0, "pass_rate": 93.33333333333333, "judge_mean": 4.866666666666666, "agent_gross_total": 2.92111993, "agent_gross_per_task": 0.19474132866666669, "judge_gross_total": 1.8535245, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 68.466, "turn_p95_seconds": 144.59359999999998, "task_p50_seconds": 75.871, "task_p95_seconds": 155.99309999999997, "tokens_per_task": 309749.6, "tokens_per_turn": 273308.4705882353, "gross_cost_per_turn": 0.17183058411764707, "main_calls": 77, "main_call_p50_seconds": 8.537, "tool_calls_per_task": 12.066666666666666, "timed_main_calls": 77, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 15, "passed": 12, "failed": 3, "errors": 0, "pass_rate": 80.0, "judge_mean": 4.533333333333333, "agent_gross_total": 1.99257217, "agent_gross_per_task": 0.13283814466666669, "judge_gross_total": 1.715922, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 33.52, "turn_p95_seconds": 84.823, "task_p50_seconds": 46.479, "task_p95_seconds": 86.10349999999998, "tokens_per_task": 258887.0, "tokens_per_turn": 228429.70588235295, "gross_cost_per_turn": 0.11721012764705883, "main_calls": 63, "main_call_p50_seconds": 3.469, "tool_calls_per_task": 9.466666666666667, "timed_main_calls": 63, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 15, "passed": 14, "failed": 1, "errors": 0, "pass_rate": 93.33333333333333, "judge_mean": 4.866666666666666, "agent_gross_total": 2.1590826, "agent_gross_per_task": 0.14393884, "judge_gross_total": 1.7489385, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 38.618, "turn_p95_seconds": 95.34839999999997, "task_p50_seconds": 52.625, "task_p95_seconds": 100.19109999999995, "tokens_per_task": 253063.73333333334, "tokens_per_turn": 223291.5294117647, "gross_cost_per_turn": 0.1270048588235294, "main_calls": 57, "main_call_p50_seconds": 4.65, "tool_calls_per_task": 9.866666666666667, "timed_main_calls": 57, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 15, "passed": 10, "failed": 5, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.133333333333334, "agent_gross_total": 2.862101945, "agent_gross_per_task": 0.19080679633333333, "judge_gross_total": 1.8115725, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 90.132, "turn_p95_seconds": 201.88579999999996, "task_p50_seconds": 111.885, "task_p95_seconds": 204.76119999999997, "tokens_per_task": 285735.2, "tokens_per_turn": 252119.29411764705, "gross_cost_per_turn": 0.16835893794117648, "main_calls": 70, "main_call_p50_seconds": 13.132000000000001, "tool_calls_per_task": 10.6, "timed_main_calls": 70, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 15, "passed": 12, "failed": 3, "errors": 0, "pass_rate": 80.0, "judge_mean": 4.466666666666667, "agent_gross_total": 2.076747875, "agent_gross_per_task": 0.13844985833333334, "judge_gross_total": 1.3603275, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 62.82, "turn_p95_seconds": 138.58859999999999, "task_p50_seconds": 46.711, "task_p95_seconds": 173.70689999999993, "tokens_per_task": 234968.53333333333, "tokens_per_turn": 207325.17647058822, "gross_cost_per_turn": 0.12216163970588235, "main_calls": 60, "main_call_p50_seconds": 7.4239999999999995, "tool_calls_per_task": 8.066666666666666, "timed_main_calls": 60, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 15, "passed": 12, "failed": 3, "errors": 0, "pass_rate": 80.0, "judge_mean": 4.466666666666667, "agent_gross_total": 2.829429975, "agent_gross_per_task": 0.188628665, "judge_gross_total": 1.957437, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 51.951, "turn_p95_seconds": 127.4408, "task_p50_seconds": 61.510000000000005, "task_p95_seconds": 136.3739, "tokens_per_task": 306980.6, "tokens_per_turn": 270865.23529411765, "gross_cost_per_turn": 0.16643705735294118, "main_calls": 73, "main_call_p50_seconds": 5.846, "tool_calls_per_task": 10.933333333333334, "timed_main_calls": 73, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 15, "passed": 12, "failed": 3, "errors": 0, "pass_rate": 80.0, "judge_mean": 4.666666666666667, "agent_gross_total": 2.435139365, "agent_gross_per_task": 0.16234262433333332, "judge_gross_total": 1.6663515, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 41.206, "turn_p95_seconds": 123.79839999999992, "task_p50_seconds": 62.444, "task_p95_seconds": 135.21459999999988, "tokens_per_task": 272192.73333333334, "tokens_per_turn": 240170.0588235294, "gross_cost_per_turn": 0.14324349205882353, "main_calls": 60, "main_call_p50_seconds": 6.4305, "tool_calls_per_task": 11.4, "timed_main_calls": 60, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 15, "passed": 12, "failed": 3, "errors": 0, "pass_rate": 80.0, "judge_mean": 4.533333333333333, "agent_gross_total": 1.86667437, "agent_gross_per_task": 0.124444958, "judge_gross_total": 1.480125, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 35.826, "turn_p95_seconds": 101.65619999999998, "task_p50_seconds": 42.757999999999996, "task_p95_seconds": 103.70179999999998, "tokens_per_task": 212895.26666666666, "tokens_per_turn": 187848.76470588235, "gross_cost_per_turn": 0.10980437470588235, "main_calls": 47, "main_call_p50_seconds": 4.285, "tool_calls_per_task": 7.933333333333334, "timed_main_calls": 47, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 15, "passed": 13, "failed": 2, "errors": 0, "pass_rate": 86.66666666666667, "judge_mean": 4.666666666666667, "agent_gross_total": 1.811467905, "agent_gross_per_task": 0.120764527, "judge_gross_total": 1.3806645, "simulator_gross_total": 0, "turn_count": 17, "turn_p50_seconds": 35.63, "turn_p95_seconds": 78.3246, "task_p50_seconds": 37.086, "task_p95_seconds": 89.40809999999996, "tokens_per_task": 201869.86666666667, "tokens_per_turn": 178120.4705882353, "gross_cost_per_turn": 0.1065569355882353, "main_calls": 44, "main_call_p50_seconds": 6.6754999999999995, "tool_calls_per_task": 7.866666666666666, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 } }, "hallucination/policy": { "argon_high": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.774193548387097, "agent_gross_total": 11.653519375, "agent_gross_per_task": 0.3759199798387097, "judge_gross_total": 1.6354635, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 47.625, "turn_p95_seconds": 142.43354999999994, "task_p50_seconds": 59.466, "task_p95_seconds": 220.7975, "tokens_per_task": 237333.96774193548, "tokens_per_turn": 175175.07142857142, "gross_cost_per_turn": 0.2774647470238095, "main_calls": 137, "main_call_p50_seconds": 7.598, "tool_calls_per_task": 8.096774193548388, "timed_main_calls": 135, "native_main_timing_omitted_requests": 2 }, "argon_low": { "observations": 31, "passed": 28, "failed": 3, "errors": 0, "pass_rate": 90.3225806451613, "judge_mean": 4.903225806451613, "agent_gross_total": 9.47357688, "agent_gross_per_task": 0.3055992541935484, "judge_gross_total": 1.857102, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 29.584, "turn_p95_seconds": 101.7945499999998, "task_p50_seconds": 33.511, "task_p95_seconds": 148.3675, "tokens_per_task": 303376.5483870968, "tokens_per_turn": 223920.7857142857, "gross_cost_per_turn": 0.22556135428571428, "main_calls": 176, "main_call_p50_seconds": 3.7965, "tool_calls_per_task": 10.161290322580646, "timed_main_calls": 176, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 31, "passed": 29, "failed": 2, "errors": 0, "pass_rate": 93.54838709677419, "judge_mean": 5.0, "agent_gross_total": 9.121716205, "agent_gross_per_task": 0.29424890983870966, "judge_gross_total": 1.4994885, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 28.1875, "turn_p95_seconds": 53.95449999999998, "task_p50_seconds": 35.938, "task_p95_seconds": 80.95949999999999, "tokens_per_task": 213287.70967741936, "tokens_per_turn": 157426.64285714287, "gross_cost_per_turn": 0.21718371916666668, "main_calls": 123, "main_call_p50_seconds": 3.832, "tool_calls_per_task": 7.161290322580645, "timed_main_calls": 123, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.645161290322581, "agent_gross_total": 5.50952743, "agent_gross_per_task": 0.1777266912903226, "judge_gross_total": 1.4100915, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 64.411, "turn_p95_seconds": 121.04654999999998, "task_p50_seconds": 74.508, "task_p95_seconds": 209.27800000000002, "tokens_per_task": 225889.29032258064, "tokens_per_turn": 166727.80952380953, "gross_cost_per_turn": 0.13117922452380953, "main_calls": 147, "main_call_p50_seconds": 9.335, "tool_calls_per_task": 7.096774193548387, "timed_main_calls": 147, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.709677419354839, "agent_gross_total": 3.479043355, "agent_gross_per_task": 0.112227205, "judge_gross_total": 1.2657615, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 27.122, "turn_p95_seconds": 79.47719999999994, "task_p50_seconds": 31.622, "task_p95_seconds": 91.7175, "tokens_per_task": 180486.38709677418, "tokens_per_turn": 133216.14285714287, "gross_cost_per_turn": 0.08283436559523809, "main_calls": 113, "main_call_p50_seconds": 3.47, "tool_calls_per_task": 5.064516129032258, "timed_main_calls": 113, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 31, "passed": 28, "failed": 3, "errors": 0, "pass_rate": 90.3225806451613, "judge_mean": 5.0, "agent_gross_total": 3.951954165, "agent_gross_per_task": 0.12748239241935483, "judge_gross_total": 1.32015, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 35.774, "turn_p95_seconds": 78.33554999999997, "task_p50_seconds": 41.246, "task_p95_seconds": 96.414, "tokens_per_task": 188783.12903225806, "tokens_per_turn": 139339.92857142858, "gross_cost_per_turn": 0.09409414678571429, "main_calls": 119, "main_call_p50_seconds": 5.551, "tool_calls_per_task": 5.483870967741935, "timed_main_calls": 119, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.709677419354839, "agent_gross_total": 5.0970796, "agent_gross_per_task": 0.16442192258064517, "judge_gross_total": 1.3828529999999999, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 90.202, "turn_p95_seconds": 196.93394999999998, "task_p50_seconds": 124.626, "task_p95_seconds": 280.4235, "tokens_per_task": 211872.48387096773, "tokens_per_turn": 156382.07142857142, "gross_cost_per_turn": 0.1213590380952381, "main_calls": 138, "main_call_p50_seconds": 19.621499999999997, "tool_calls_per_task": 6.032258064516129, "timed_main_calls": 138, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.645161290322581, "agent_gross_total": 3.72807012, "agent_gross_per_task": 0.1202603264516129, "judge_gross_total": 1.212798, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 51.8425, "turn_p95_seconds": 197.39969999999997, "task_p50_seconds": 51.294, "task_p95_seconds": 306.036, "tokens_per_task": 181340.12903225806, "tokens_per_turn": 133846.2857142857, "gross_cost_per_turn": 0.08876357428571428, "main_calls": 120, "main_call_p50_seconds": 11.2395, "tool_calls_per_task": 4.870967741935484, "timed_main_calls": 120, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.580645161290323, "agent_gross_total": 4.54430512, "agent_gross_per_task": 0.14659048774193548, "judge_gross_total": 1.3365705, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 42.696, "turn_p95_seconds": 134.62519999999995, "task_p50_seconds": 50.307, "task_p95_seconds": 201.3785, "tokens_per_task": 203437.61290322582, "tokens_per_turn": 150156.33333333334, "gross_cost_per_turn": 0.10819774095238095, "main_calls": 133, "main_call_p50_seconds": 7.047, "tool_calls_per_task": 6.096774193548387, "timed_main_calls": 133, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 31, "passed": 27, "failed": 4, "errors": 0, "pass_rate": 87.09677419354838, "judge_mean": 4.806451612903226, "agent_gross_total": 4.148319945, "agent_gross_per_task": 0.13381677241935483, "judge_gross_total": 1.3479165, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 34.204, "turn_p95_seconds": 75.00609999999999, "task_p50_seconds": 42.171, "task_p95_seconds": 120.17150000000001, "tokens_per_task": 204231.93548387097, "tokens_per_turn": 150742.61904761905, "gross_cost_per_turn": 0.0987695225, "main_calls": 126, "main_call_p50_seconds": 6.0025, "tool_calls_per_task": 6.67741935483871, "timed_main_calls": 126, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 31, "passed": 26, "failed": 5, "errors": 0, "pass_rate": 83.87096774193549, "judge_mean": 4.741935483870968, "agent_gross_total": 3.618542645, "agent_gross_per_task": 0.11672718209677418, "judge_gross_total": 1.377063, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 30.0715, "turn_p95_seconds": 97.26279999999997, "task_p50_seconds": 35.826, "task_p95_seconds": 111.05199999999999, "tokens_per_task": 187017.29032258064, "tokens_per_turn": 138036.57142857142, "gross_cost_per_turn": 0.08615577726190475, "main_calls": 113, "main_call_p50_seconds": 4.257, "tool_calls_per_task": 5.354838709677419, "timed_main_calls": 113, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 31, "passed": 28, "failed": 3, "errors": 0, "pass_rate": 90.3225806451613, "judge_mean": 4.774193548387097, "agent_gross_total": 3.3700702, "agent_gross_per_task": 0.10871194193548386, "judge_gross_total": 1.13064, "simulator_gross_total": 0, "turn_count": 42, "turn_p50_seconds": 31.543, "turn_p95_seconds": 89.35514999999998, "task_p50_seconds": 40.057, "task_p95_seconds": 147.78199999999998, "tokens_per_task": 165156.8064516129, "tokens_per_turn": 121901.45238095238, "gross_cost_per_turn": 0.08023976666666666, "main_calls": 101, "main_call_p50_seconds": 6.449, "tool_calls_per_task": 4.516129032258065, "timed_main_calls": 101, "native_main_timing_omitted_requests": 0 } }, "returns/exchanges": { "argon_high": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 5.0, "agent_gross_total": 4.6802063, "agent_gross_per_task": 0.5200229222222222, "judge_gross_total": 0.5232345, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 50.5395, "turn_p95_seconds": 127.27575, "task_p50_seconds": 110.582, "task_p95_seconds": 240.06139999999996, "tokens_per_task": 328728.55555555556, "tokens_per_turn": 184909.8125, "gross_cost_per_turn": 0.29251289375, "main_calls": 58, "main_call_p50_seconds": 6.693, "tool_calls_per_task": 8.11111111111111, "timed_main_calls": 57, "native_main_timing_omitted_requests": 1 }, "argon_low": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 4.666666666666667, "agent_gross_total": 4.11460273, "agent_gross_per_task": 0.4571780811111111, "judge_gross_total": 0.5332439999999999, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 30.1485, "turn_p95_seconds": 120.55825, "task_p50_seconds": 39.562, "task_p95_seconds": 218.82459999999992, "tokens_per_task": 400981.8888888889, "tokens_per_turn": 225552.3125, "gross_cost_per_turn": 0.257162670625, "main_calls": 80, "main_call_p50_seconds": 3.7264999999999997, "tool_calls_per_task": 10.777777777777779, "timed_main_calls": 80, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 9, "passed": 9, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 4.326154115, "agent_gross_per_task": 0.4806837905555555, "judge_gross_total": 0.516012, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 31.601, "turn_p95_seconds": 99.1685, "task_p50_seconds": 41.848, "task_p95_seconds": 165.75719999999995, "tokens_per_task": 314849.3333333333, "tokens_per_turn": 177102.75, "gross_cost_per_turn": 0.2703846321875, "main_calls": 56, "main_call_p50_seconds": 3.6014999999999997, "tool_calls_per_task": 8.666666666666666, "timed_main_calls": 56, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 9, "passed": 7, "failed": 2, "errors": 0, "pass_rate": 77.77777777777777, "judge_mean": 4.555555555555555, "agent_gross_total": 2.157358815, "agent_gross_per_task": 0.23970653499999997, "judge_gross_total": 0.45074400000000003, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 67.5505, "turn_p95_seconds": 115.062, "task_p50_seconds": 107.38, "task_p95_seconds": 274.24339999999995, "tokens_per_task": 286854.22222222225, "tokens_per_turn": 161355.5, "gross_cost_per_turn": 0.1348349259375, "main_calls": 52, "main_call_p50_seconds": 11.207, "tool_calls_per_task": 7.0, "timed_main_calls": 52, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 9, "passed": 9, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.407725265, "agent_gross_per_task": 0.15641391833333335, "judge_gross_total": 0.3692235, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 30.942500000000003, "turn_p95_seconds": 86.7575, "task_p50_seconds": 33.65, "task_p95_seconds": 158.47079999999994, "tokens_per_task": 231131.22222222222, "tokens_per_turn": 130011.3125, "gross_cost_per_turn": 0.0879828290625, "main_calls": 42, "main_call_p50_seconds": 3.4285, "tool_calls_per_task": 4.555555555555555, "timed_main_calls": 42, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 5.0, "agent_gross_total": 1.621017125, "agent_gross_per_task": 0.1801130138888889, "judge_gross_total": 0.3954375, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 37.6695, "turn_p95_seconds": 101.8695, "task_p50_seconds": 78.64, "task_p95_seconds": 182.70119999999997, "tokens_per_task": 249286.55555555556, "tokens_per_turn": 140223.6875, "gross_cost_per_turn": 0.1013135703125, "main_calls": 46, "main_call_p50_seconds": 7.4005, "tool_calls_per_task": 5.222222222222222, "timed_main_calls": 46, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 5.0, "agent_gross_total": 2.00549609, "agent_gross_per_task": 0.22283289888888888, "judge_gross_total": 0.385776, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 85.3965, "turn_p95_seconds": 173.3255, "task_p50_seconds": 166.465, "task_p95_seconds": 283.8216, "tokens_per_task": 251464.77777777778, "tokens_per_turn": 141448.9375, "gross_cost_per_turn": 0.125343505625, "main_calls": 45, "main_call_p50_seconds": 20.358, "tool_calls_per_task": 5.111111111111111, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 9, "passed": 9, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.511690635, "agent_gross_per_task": 0.1679656261111111, "judge_gross_total": 0.3609315, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 79.49000000000001, "turn_p95_seconds": 205.23874999999998, "task_p50_seconds": 227.927, "task_p95_seconds": 333.47159999999997, "tokens_per_task": 231796.77777777778, "tokens_per_turn": 130385.6875, "gross_cost_per_turn": 0.0944806646875, "main_calls": 45, "main_call_p50_seconds": 20.748, "tool_calls_per_task": 4.777777777777778, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 4.555555555555555, "agent_gross_total": 1.767909615, "agent_gross_per_task": 0.19643440166666667, "judge_gross_total": 0.3774675, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 41.7475, "turn_p95_seconds": 139.68900000000002, "task_p50_seconds": 50.307, "task_p95_seconds": 243.2428, "tokens_per_task": 236675.66666666666, "tokens_per_turn": 133130.0625, "gross_cost_per_turn": 0.1104943509375, "main_calls": 43, "main_call_p50_seconds": 6.95, "tool_calls_per_task": 4.888888888888889, "timed_main_calls": 43, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 5.0, "agent_gross_total": 1.7591831, "agent_gross_per_task": 0.19546478888888888, "judge_gross_total": 0.439224, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 36.9755, "turn_p95_seconds": 104.82124999999999, "task_p50_seconds": 73.013, "task_p95_seconds": 199.04159999999996, "tokens_per_task": 287217.1111111111, "tokens_per_turn": 161559.625, "gross_cost_per_turn": 0.10994894375, "main_calls": 52, "main_call_p50_seconds": 6.551, "tool_calls_per_task": 7.0, "timed_main_calls": 52, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 9, "passed": 8, "failed": 1, "errors": 0, "pass_rate": 88.88888888888889, "judge_mean": 5.0, "agent_gross_total": 1.49028954, "agent_gross_per_task": 0.16558772666666666, "judge_gross_total": 0.4478685, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 30.572499999999998, "turn_p95_seconds": 97.71000000000001, "task_p50_seconds": 55.08, "task_p95_seconds": 173.65399999999994, "tokens_per_task": 257533.55555555556, "tokens_per_turn": 144862.625, "gross_cost_per_turn": 0.09314309625, "main_calls": 45, "main_call_p50_seconds": 4.061, "tool_calls_per_task": 5.555555555555555, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 9, "passed": 9, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.341901285, "agent_gross_per_task": 0.14910014277777778, "judge_gross_total": 0.3625395, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 36.3555, "turn_p95_seconds": 87.00524999999999, "task_p50_seconds": 72.711, "task_p95_seconds": 157.57119999999998, "tokens_per_task": 224863.44444444444, "tokens_per_turn": 126485.6875, "gross_cost_per_turn": 0.0838688303125, "main_calls": 41, "main_call_p50_seconds": 4.767, "tool_calls_per_task": 4.555555555555555, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 } }, "shopping/cross-sell": { "argon_high": { "observations": 11, "passed": 10, "failed": 1, "errors": 0, "pass_rate": 90.9090909090909, "judge_mean": 4.818181818181818, "agent_gross_total": 5.5326508500000005, "agent_gross_per_task": 0.5029682590909091, "judge_gross_total": 1.293054, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 79.3845, "turn_p95_seconds": 144.7545, "task_p50_seconds": 84.513, "task_p95_seconds": 190.82850000000002, "tokens_per_task": 308078.54545454547, "tokens_per_turn": 242061.7142857143, "gross_cost_per_turn": 0.39518934642857145, "main_calls": 50, "main_call_p50_seconds": 7.853, "tool_calls_per_task": 12.363636363636363, "timed_main_calls": 49, "native_main_timing_omitted_requests": 1 }, "argon_low": { "observations": 11, "passed": 10, "failed": 1, "errors": 0, "pass_rate": 90.9090909090909, "judge_mean": 4.818181818181818, "agent_gross_total": 7.36392832, "agent_gross_per_task": 0.6694480290909092, "judge_gross_total": 3.7623405, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 37.492, "turn_p95_seconds": 303.7673, "task_p50_seconds": 51.486999999999995, "task_p95_seconds": 315.91700000000003, "tokens_per_task": 852243.6363636364, "tokens_per_turn": 669620.0, "gross_cost_per_turn": 0.52599488, "main_calls": 82, "main_call_p50_seconds": 4.0954999999999995, "tool_calls_per_task": 21.818181818181817, "timed_main_calls": 82, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 11, "passed": 11, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 5.494972185, "agent_gross_per_task": 0.4995429259090909, "judge_gross_total": 2.129385, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 35.6195, "turn_p95_seconds": 102.38004999999998, "task_p50_seconds": 54.212, "task_p95_seconds": 112.0555, "tokens_per_task": 393474.8181818182, "tokens_per_turn": 309158.78571428574, "gross_cost_per_turn": 0.39249801321428573, "main_calls": 53, "main_call_p50_seconds": 3.952, "tool_calls_per_task": 12.454545454545455, "timed_main_calls": 53, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 11, "passed": 10, "failed": 1, "errors": 0, "pass_rate": 90.9090909090909, "judge_mean": 4.818181818181818, "agent_gross_total": 2.42904379, "agent_gross_per_task": 0.22082216272727273, "judge_gross_total": 1.710264, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 70.9225, "turn_p95_seconds": 151.7378, "task_p50_seconds": 90.706, "task_p95_seconds": 163.6225, "tokens_per_task": 365200.54545454547, "tokens_per_turn": 286943.28571428574, "gross_cost_per_turn": 0.17350312785714286, "main_calls": 64, "main_call_p50_seconds": 8.9285, "tool_calls_per_task": 12.818181818181818, "timed_main_calls": 64, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 11, "passed": 9, "failed": 2, "errors": 0, "pass_rate": 81.81818181818181, "judge_mean": 4.545454545454546, "agent_gross_total": 1.71154047, "agent_gross_per_task": 0.15559458818181818, "judge_gross_total": 1.648662, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 39.9995, "turn_p95_seconds": 86.74374999999999, "task_p50_seconds": 61.288000000000004, "task_p95_seconds": 88.6645, "tokens_per_task": 313874.7272727273, "tokens_per_turn": 246615.85714285713, "gross_cost_per_turn": 0.12225289071428572, "main_calls": 52, "main_call_p50_seconds": 3.551, "tool_calls_per_task": 10.272727272727273, "timed_main_calls": 52, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 11, "passed": 10, "failed": 1, "errors": 0, "pass_rate": 90.9090909090909, "judge_mean": 4.818181818181818, "agent_gross_total": 1.866654765, "agent_gross_per_task": 0.16969588772727273, "judge_gross_total": 1.5825195, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 37.862, "turn_p95_seconds": 102.61244999999998, "task_p50_seconds": 64.056, "task_p95_seconds": 110.43100000000001, "tokens_per_task": 304669.0909090909, "tokens_per_turn": 239382.85714285713, "gross_cost_per_turn": 0.13333248321428573, "main_calls": 46, "main_call_p50_seconds": 4.8355, "tool_calls_per_task": 10.363636363636363, "timed_main_calls": 46, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 11, "passed": 7, "failed": 4, "errors": 0, "pass_rate": 63.63636363636363, "judge_mean": 4.090909090909091, "agent_gross_total": 2.488420045, "agent_gross_per_task": 0.22622000409090906, "judge_gross_total": 1.6506255, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 92.693, "turn_p95_seconds": 206.19889999999998, "task_p50_seconds": 116.90899999999999, "task_p95_seconds": 210.512, "tokens_per_task": 330067.8181818182, "tokens_per_turn": 259339.0, "gross_cost_per_turn": 0.17774428892857141, "main_calls": 54, "main_call_p50_seconds": 12.6785, "tool_calls_per_task": 11.272727272727273, "timed_main_calls": 54, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 11, "passed": 10, "failed": 1, "errors": 0, "pass_rate": 90.9090909090909, "judge_mean": 4.818181818181818, "agent_gross_total": 1.81643636, "agent_gross_per_task": 0.16513057818181817, "judge_gross_total": 1.26159, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 70.92949999999999, "turn_p95_seconds": 167.82139999999998, "task_p50_seconds": 73.797, "task_p95_seconds": 212.825, "tokens_per_task": 277814.1818181818, "tokens_per_turn": 218282.57142857142, "gross_cost_per_turn": 0.1297454542857143, "main_calls": 48, "main_call_p50_seconds": 11.039, "tool_calls_per_task": 8.0, "timed_main_calls": 48, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 11, "passed": 9, "failed": 2, "errors": 0, "pass_rate": 81.81818181818181, "judge_mean": 4.545454545454546, "agent_gross_total": 2.5105077749999998, "agent_gross_per_task": 0.2282279795454545, "judge_gross_total": 1.8490965000000001, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 50.998000000000005, "turn_p95_seconds": 127.92814999999999, "task_p50_seconds": 64.116, "task_p95_seconds": 140.5965, "tokens_per_task": 358997.8181818182, "tokens_per_turn": 282069.71428571426, "gross_cost_per_turn": 0.1793219839285714, "main_calls": 58, "main_call_p50_seconds": 6.564500000000001, "tool_calls_per_task": 11.545454545454545, "timed_main_calls": 58, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 11, "passed": 8, "failed": 3, "errors": 0, "pass_rate": 72.72727272727273, "judge_mean": 4.545454545454546, "agent_gross_total": 2.081679235, "agent_gross_per_task": 0.18924356681818183, "judge_gross_total": 1.4873445, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 39.6485, "turn_p95_seconds": 140.92269999999996, "task_p50_seconds": 70.955, "task_p95_seconds": 158.047, "tokens_per_task": 324669.0, "tokens_per_turn": 255097.07142857142, "gross_cost_per_turn": 0.14869137392857143, "main_calls": 47, "main_call_p50_seconds": 6.236, "tool_calls_per_task": 13.0, "timed_main_calls": 47, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 11, "passed": 8, "failed": 3, "errors": 0, "pass_rate": 72.72727272727273, "judge_mean": 4.454545454545454, "agent_gross_total": 1.51185134, "agent_gross_per_task": 0.1374410309090909, "judge_gross_total": 1.2486825, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 32.9275, "turn_p95_seconds": 104.7246, "task_p50_seconds": 54.673, "task_p95_seconds": 107.793, "tokens_per_task": 243536.63636363635, "tokens_per_turn": 191350.2142857143, "gross_cost_per_turn": 0.10798938142857142, "main_calls": 35, "main_call_p50_seconds": 4.265, "tool_calls_per_task": 7.636363636363637, "timed_main_calls": 35, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 11, "passed": 9, "failed": 2, "errors": 0, "pass_rate": 81.81818181818181, "judge_mean": 4.545454545454546, "agent_gross_total": 1.49308559, "agent_gross_per_task": 0.13573505363636362, "judge_gross_total": 1.2394905, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 36.545, "turn_p95_seconds": 104.56359999999998, "task_p50_seconds": 48.017, "task_p95_seconds": 140.58499999999998, "tokens_per_task": 238889.81818181818, "tokens_per_turn": 187699.14285714287, "gross_cost_per_turn": 0.10664897071428571, "main_calls": 37, "main_call_p50_seconds": 8.432, "tool_calls_per_task": 8.181818181818182, "timed_main_calls": 37, "native_main_timing_omitted_requests": 0 } }, "support/escalation": { "argon_high": { "observations": 8, "passed": 4, "failed": 4, "errors": 0, "pass_rate": 50.0, "judge_mean": 4.125, "agent_gross_total": 2.5000140749999997, "agent_gross_per_task": 0.31250175937499997, "judge_gross_total": 0.50844, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 44.7735, "turn_p95_seconds": 101.88454999999996, "task_p50_seconds": 45.3115, "task_p95_seconds": 119.5773, "tokens_per_task": 211103.0, "tokens_per_turn": 168882.4, "gross_cost_per_turn": 0.2500014075, "main_calls": 28, "main_call_p50_seconds": 8.189, "tool_calls_per_task": 6.875, "timed_main_calls": 28, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 8, "passed": 6, "failed": 2, "errors": 0, "pass_rate": 75.0, "judge_mean": 5.0, "agent_gross_total": 1.529425605, "agent_gross_per_task": 0.191178200625, "judge_gross_total": 0.423576, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 25.7845, "turn_p95_seconds": 41.325449999999975, "task_p50_seconds": 29.23, "task_p95_seconds": 63.41279999999999, "tokens_per_task": 176849.125, "tokens_per_turn": 141479.3, "gross_cost_per_turn": 0.1529425605, "main_calls": 24, "main_call_p50_seconds": 4.0725, "tool_calls_per_task": 5.75, "timed_main_calls": 24, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 8, "passed": 6, "failed": 2, "errors": 0, "pass_rate": 75.0, "judge_mean": 5.0, "agent_gross_total": 1.91648268, "agent_gross_per_task": 0.239560335, "judge_gross_total": 0.6128295, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 26.1005, "turn_p95_seconds": 60.408099999999955, "task_p50_seconds": 31.997500000000002, "task_p95_seconds": 77.97735, "tokens_per_task": 194024.875, "tokens_per_turn": 155219.9, "gross_cost_per_turn": 0.191648268, "main_calls": 25, "main_call_p50_seconds": 4.152, "tool_calls_per_task": 5.875, "timed_main_calls": 25, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.125, "agent_gross_total": 1.32291105, "agent_gross_per_task": 0.16536388125, "judge_gross_total": 0.45797550000000004, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 65.0115, "turn_p95_seconds": 130.10484999999994, "task_p50_seconds": 75.1895, "task_p95_seconds": 144.61549999999997, "tokens_per_task": 215785.5, "tokens_per_turn": 172628.4, "gross_cost_per_turn": 0.13229110500000002, "main_calls": 32, "main_call_p50_seconds": 8.8725, "tool_calls_per_task": 6.125, "timed_main_calls": 32, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.5, "agent_gross_total": 0.8144771000000001, "agent_gross_per_task": 0.10180963750000001, "judge_gross_total": 0.4418235, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 27.034, "turn_p95_seconds": 61.15449999999999, "task_p50_seconds": 27.034, "task_p95_seconds": 82.69364999999999, "tokens_per_task": 199038.375, "tokens_per_turn": 159230.7, "gross_cost_per_turn": 0.08144771, "main_calls": 30, "main_call_p50_seconds": 2.9764999999999997, "tool_calls_per_task": 5.375, "timed_main_calls": 30, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 8, "passed": 6, "failed": 2, "errors": 0, "pass_rate": 75.0, "judge_mean": 5.0, "agent_gross_total": 0.942578075, "agent_gross_per_task": 0.117822259375, "judge_gross_total": 0.462081, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 34.558, "turn_p95_seconds": 63.51714999999997, "task_p50_seconds": 42.134, "task_p95_seconds": 69.78474999999999, "tokens_per_task": 193892.25, "tokens_per_turn": 155113.8, "gross_cost_per_turn": 0.0942578075, "main_calls": 30, "main_call_p50_seconds": 4.3805, "tool_calls_per_task": 5.625, "timed_main_calls": 30, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.375, "agent_gross_total": 1.3242924, "agent_gross_per_task": 0.16553655, "judge_gross_total": 0.47131049999999997, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 94.9385, "turn_p95_seconds": 289.6043999999998, "task_p50_seconds": 118.68700000000001, "task_p95_seconds": 406.37114999999983, "tokens_per_task": 235854.75, "tokens_per_turn": 188683.8, "gross_cost_per_turn": 0.13242924, "main_calls": 42, "main_call_p50_seconds": 21.1575, "tool_calls_per_task": 6.75, "timed_main_calls": 42, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.375, "agent_gross_total": 0.872068775, "agent_gross_per_task": 0.109008596875, "judge_gross_total": 0.4251555, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 62.5455, "turn_p95_seconds": 208.14959999999982, "task_p50_seconds": 88.9145, "task_p95_seconds": 236.56509999999992, "tokens_per_task": 182896.75, "tokens_per_turn": 146317.4, "gross_cost_per_turn": 0.0872068775, "main_calls": 32, "main_call_p50_seconds": 10.682, "tool_calls_per_task": 4.5, "timed_main_calls": 32, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 8, "passed": 6, "failed": 2, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.625, "agent_gross_total": 1.31940985, "agent_gross_per_task": 0.16492623125, "judge_gross_total": 0.473766, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 52.058499999999995, "turn_p95_seconds": 138.19909999999996, "task_p50_seconds": 66.0175, "task_p95_seconds": 140.27329999999998, "tokens_per_task": 243059.25, "tokens_per_turn": 194447.4, "gross_cost_per_turn": 0.131940985, "main_calls": 38, "main_call_p50_seconds": 8.536999999999999, "tool_calls_per_task": 6.625, "timed_main_calls": 38, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.625, "agent_gross_total": 1.027911395, "agent_gross_per_task": 0.128488924375, "judge_gross_total": 0.356361, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 30.433999999999997, "turn_p95_seconds": 75.72819999999993, "task_p50_seconds": 43.179, "task_p95_seconds": 90.73199999999999, "tokens_per_task": 214547.375, "tokens_per_turn": 171637.9, "gross_cost_per_turn": 0.1027911395, "main_calls": 30, "main_call_p50_seconds": 5.2165, "tool_calls_per_task": 6.375, "timed_main_calls": 30, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 8, "passed": 5, "failed": 3, "errors": 0, "pass_rate": 62.5, "judge_mean": 4.75, "agent_gross_total": 0.9809391749999999, "agent_gross_per_task": 0.12261739687499999, "judge_gross_total": 0.43788900000000003, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 28.695999999999998, "turn_p95_seconds": 121.60589999999999, "task_p50_seconds": 44.277, "task_p95_seconds": 122.2577, "tokens_per_task": 199194.5, "tokens_per_turn": 159355.6, "gross_cost_per_turn": 0.09809391749999999, "main_calls": 30, "main_call_p50_seconds": 4.233499999999999, "tool_calls_per_task": 4.625, "timed_main_calls": 30, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 8, "passed": 6, "failed": 2, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.5, "agent_gross_total": 0.8070545, "agent_gross_per_task": 0.1008818125, "judge_gross_total": 0.434751, "simulator_gross_total": 0, "turn_count": 10, "turn_p50_seconds": 34.723, "turn_p95_seconds": 106.60764999999994, "task_p50_seconds": 47.692, "task_p95_seconds": 117.09754999999998, "tokens_per_task": 165455.875, "tokens_per_turn": 132364.7, "gross_cost_per_turn": 0.08070545, "main_calls": 24, "main_call_p50_seconds": 6.685499999999999, "tool_calls_per_task": 4.875, "timed_main_calls": 24, "native_main_timing_omitted_requests": 0 } }, "tone/transcript": { "argon_high": { "observations": 13, "passed": 8, "failed": 5, "errors": 0, "pass_rate": 61.53846153846154, "judge_mean": 4.461538461538462, "agent_gross_total": 3.866354775, "agent_gross_per_task": 0.29741190576923077, "judge_gross_total": 0.4772025, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 44.7735, "turn_p95_seconds": 94.14725, "task_p50_seconds": 45.468, "task_p95_seconds": 118.42179999999999, "tokens_per_task": 177073.23076923078, "tokens_per_turn": 143872.0, "gross_cost_per_turn": 0.2416471734375, "main_calls": 44, "main_call_p50_seconds": 8.428, "tool_calls_per_task": 5.384615384615385, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 13, "passed": 11, "failed": 2, "errors": 0, "pass_rate": 84.61538461538461, "judge_mean": 5.0, "agent_gross_total": 2.35662084, "agent_gross_per_task": 0.18127852615384615, "judge_gross_total": 0.34398, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 24.4655, "turn_p95_seconds": 41.449, "task_p50_seconds": 29.226, "task_p95_seconds": 58.96379999999997, "tokens_per_task": 153241.15384615384, "tokens_per_turn": 124508.4375, "gross_cost_per_turn": 0.1472888025, "main_calls": 41, "main_call_p50_seconds": 3.914, "tool_calls_per_task": 4.384615384615385, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 13, "passed": 11, "failed": 2, "errors": 0, "pass_rate": 84.61538461538461, "judge_mean": 5.0, "agent_gross_total": 2.84003319, "agent_gross_per_task": 0.21846409153846155, "judge_gross_total": 0.5086005, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 24.033, "turn_p95_seconds": 43.6515, "task_p50_seconds": 26.483, "task_p95_seconds": 67.24519999999998, "tokens_per_task": 179806.53846153847, "tokens_per_turn": 146092.8125, "gross_cost_per_turn": 0.177502074375, "main_calls": 45, "main_call_p50_seconds": 3.554, "tool_calls_per_task": 5.6923076923076925, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 13, "passed": 9, "failed": 4, "errors": 0, "pass_rate": 69.23076923076923, "judge_mean": 4.153846153846154, "agent_gross_total": 2.116527935, "agent_gross_per_task": 0.16280984115384617, "judge_gross_total": 0.4610325, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 65.063, "turn_p95_seconds": 162.21075, "task_p50_seconds": 84.183, "task_p95_seconds": 167.71319999999994, "tokens_per_task": 197893.84615384616, "tokens_per_turn": 160788.75, "gross_cost_per_turn": 0.1322829959375, "main_calls": 57, "main_call_p50_seconds": 9.478, "tool_calls_per_task": 5.923076923076923, "timed_main_calls": 57, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 13, "passed": 10, "failed": 3, "errors": 0, "pass_rate": 76.92307692307692, "judge_mean": 4.6923076923076925, "agent_gross_total": 1.201265635, "agent_gross_per_task": 0.09240504884615384, "judge_gross_total": 0.442086, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 27.034, "turn_p95_seconds": 65.946, "task_p50_seconds": 28.88, "task_p95_seconds": 80.93459999999996, "tokens_per_task": 151427.15384615384, "tokens_per_turn": 123034.5625, "gross_cost_per_turn": 0.0750791021875, "main_calls": 41, "main_call_p50_seconds": 3.509, "tool_calls_per_task": 3.8461538461538463, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 13, "passed": 10, "failed": 3, "errors": 0, "pass_rate": 76.92307692307692, "judge_mean": 5.0, "agent_gross_total": 1.43001374, "agent_gross_per_task": 0.11000105692307692, "judge_gross_total": 0.417333, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 31.397999999999996, "turn_p95_seconds": 74.07325, "task_p50_seconds": 38.498, "task_p95_seconds": 81.8928, "tokens_per_task": 160350.0, "tokens_per_turn": 130284.375, "gross_cost_per_turn": 0.08937585875, "main_calls": 45, "main_call_p50_seconds": 5.551, "tool_calls_per_task": 4.0, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 13, "passed": 11, "failed": 2, "errors": 0, "pass_rate": 84.61538461538461, "judge_mean": 4.769230769230769, "agent_gross_total": 2.032961, "agent_gross_per_task": 0.15638161538461537, "judge_gross_total": 0.47494949999999997, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 92.693, "turn_p95_seconds": 247.332, "task_p50_seconds": 116.90899999999999, "task_p95_seconds": 331.6533999999996, "tokens_per_task": 197592.61538461538, "tokens_per_turn": 160544.0, "gross_cost_per_turn": 0.1270600625, "main_calls": 58, "main_call_p50_seconds": 22.4315, "tool_calls_per_task": 5.153846153846154, "timed_main_calls": 58, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 13, "passed": 10, "failed": 3, "errors": 0, "pass_rate": 76.92307692307692, "judge_mean": 4.615384615384615, "agent_gross_total": 1.324112435, "agent_gross_per_task": 0.10185480269230769, "judge_gross_total": 0.43080599999999997, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 57.1105, "turn_p95_seconds": 242.42175, "task_p50_seconds": 104.032, "task_p95_seconds": 251.1185999999999, "tokens_per_task": 153002.53846153847, "tokens_per_turn": 124314.5625, "gross_cost_per_turn": 0.0827570271875, "main_calls": 44, "main_call_p50_seconds": 21.431, "tool_calls_per_task": 3.4615384615384617, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 13, "passed": 11, "failed": 2, "errors": 0, "pass_rate": 84.61538461538461, "judge_mean": 4.769230769230769, "agent_gross_total": 1.6413589, "agent_gross_per_task": 0.12625837692307693, "judge_gross_total": 0.461037, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 43.94, "turn_p95_seconds": 88.86125, "task_p50_seconds": 50.307, "task_p95_seconds": 121.44679999999994, "tokens_per_task": 169399.15384615384, "tokens_per_turn": 137636.8125, "gross_cost_per_turn": 0.10258493125, "main_calls": 50, "main_call_p50_seconds": 8.536999999999999, "tool_calls_per_task": 4.615384615384615, "timed_main_calls": 50, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 13, "passed": 10, "failed": 3, "errors": 0, "pass_rate": 76.92307692307692, "judge_mean": 4.769230769230769, "agent_gross_total": 1.4303179799999999, "agent_gross_per_task": 0.11002445999999999, "judge_gross_total": 0.3921975, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 31.515, "turn_p95_seconds": 62.80325, "task_p50_seconds": 42.171, "task_p95_seconds": 74.4342, "tokens_per_task": 167719.38461538462, "tokens_per_turn": 136272.0, "gross_cost_per_turn": 0.08939487374999999, "main_calls": 46, "main_call_p50_seconds": 5.3575, "tool_calls_per_task": 5.538461538461538, "timed_main_calls": 46, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 13, "passed": 9, "failed": 4, "errors": 0, "pass_rate": 69.23076923076923, "judge_mean": 4.6923076923076925, "agent_gross_total": 1.2798367449999999, "agent_gross_per_task": 0.09844898038461537, "judge_gross_total": 0.41301899999999997, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 28.2805, "turn_p95_seconds": 75.86625000000001, "task_p50_seconds": 28.912, "task_p95_seconds": 106.26239999999996, "tokens_per_task": 159339.6923076923, "tokens_per_turn": 129463.5, "gross_cost_per_turn": 0.07998979656249999, "main_calls": 43, "main_call_p50_seconds": 4.265, "tool_calls_per_task": 3.6923076923076925, "timed_main_calls": 43, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 13, "passed": 11, "failed": 2, "errors": 0, "pass_rate": 84.61538461538461, "judge_mean": 4.6923076923076925, "agent_gross_total": 1.20020055, "agent_gross_per_task": 0.09232311923076922, "judge_gross_total": 0.3620955, "simulator_gross_total": 0, "turn_count": 16, "turn_p50_seconds": 29.273, "turn_p95_seconds": 135.51049999999998, "task_p50_seconds": 40.057, "task_p95_seconds": 144.42179999999996, "tokens_per_task": 140699.07692307694, "tokens_per_turn": 114318.0, "gross_cost_per_turn": 0.075012534375, "main_calls": 39, "main_call_p50_seconds": 8.713, "tool_calls_per_task": 3.3846153846153846, "timed_main_calls": 39, "native_main_timing_omitted_requests": 0 } }, "tool-use correctness": { "argon_high": { "observations": 23, "passed": 17, "failed": 6, "errors": 0, "pass_rate": 73.91304347826087, "judge_mean": 4.608695652173913, "agent_gross_total": 10.109161725, "agent_gross_per_task": 0.4395287706521739, "judge_gross_total": 1.6841955, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 55.153, "turn_p95_seconds": 138.6743, "task_p50_seconds": 84.513, "task_p95_seconds": 240.38749999999993, "tokens_per_task": 284325.95652173914, "tokens_per_turn": 192338.14705882352, "gross_cost_per_turn": 0.29732828602941175, "main_calls": 114, "main_call_p50_seconds": 7.532, "tool_calls_per_task": 10.0, "timed_main_calls": 111, "native_main_timing_omitted_requests": 3 }, "argon_low": { "observations": 23, "passed": 19, "failed": 4, "errors": 0, "pass_rate": 82.6086956521739, "judge_mean": 4.782608695652174, "agent_gross_total": 7.381760445, "agent_gross_per_task": 0.32094610630434783, "judge_gross_total": 1.5588795, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 29.584, "turn_p95_seconds": 73.7811499999999, "task_p50_seconds": 33.511, "task_p95_seconds": 96.87499999999999, "tokens_per_task": 286007.5652173913, "tokens_per_turn": 193475.70588235295, "gross_cost_per_turn": 0.2171106013235294, "main_calls": 125, "main_call_p50_seconds": 3.879, "tool_calls_per_task": 9.826086956521738, "timed_main_calls": 125, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 23, "passed": 21, "failed": 2, "errors": 0, "pass_rate": 91.30434782608695, "judge_mean": 5.0, "agent_gross_total": 8.18628981, "agent_gross_per_task": 0.3559256439130435, "judge_gross_total": 1.632336, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 28.881, "turn_p95_seconds": 85.80894999999997, "task_p50_seconds": 35.948, "task_p95_seconds": 82.76499999999999, "tokens_per_task": 250497.34782608695, "tokens_per_turn": 169454.08823529413, "gross_cost_per_turn": 0.24077322970588236, "main_calls": 101, "main_call_p50_seconds": 3.916, "tool_calls_per_task": 8.652173913043478, "timed_main_calls": 101, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 23, "passed": 17, "failed": 6, "errors": 0, "pass_rate": 73.91304347826087, "judge_mean": 4.434782608695652, "agent_gross_total": 4.59760292, "agent_gross_per_task": 0.19989577913043477, "judge_gross_total": 1.4444565, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 63.2555, "turn_p95_seconds": 118.52114999999999, "task_p50_seconds": 90.706, "task_p95_seconds": 221.2947999999999, "tokens_per_task": 264685.652173913, "tokens_per_turn": 179052.0588235294, "gross_cost_per_turn": 0.13522361529411764, "main_calls": 120, "main_call_p50_seconds": 8.673, "tool_calls_per_task": 8.478260869565217, "timed_main_calls": 120, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 23, "passed": 18, "failed": 5, "errors": 0, "pass_rate": 78.26086956521739, "judge_mean": 4.608695652173913, "agent_gross_total": 2.88572261, "agent_gross_per_task": 0.12546620043478263, "judge_gross_total": 1.2586935, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 27.3315, "turn_p95_seconds": 70.41139999999997, "task_p50_seconds": 31.622, "task_p95_seconds": 95.91149999999999, "tokens_per_task": 215566.08695652173, "tokens_per_turn": 145824.11764705883, "gross_cost_per_turn": 0.0848741944117647, "main_calls": 97, "main_call_p50_seconds": 3.324, "tool_calls_per_task": 6.173913043478261, "timed_main_calls": 97, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 23, "passed": 19, "failed": 4, "errors": 0, "pass_rate": 82.6086956521739, "judge_mean": 4.913043478260869, "agent_gross_total": 3.38056553, "agent_gross_per_task": 0.14698111, "judge_gross_total": 1.324329, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 36.959, "turn_p95_seconds": 85.31519999999996, "task_p50_seconds": 53.489, "task_p95_seconds": 103.78399999999996, "tokens_per_task": 222283.34782608695, "tokens_per_turn": 150368.14705882352, "gross_cost_per_turn": 0.09942839794117647, "main_calls": 100, "main_call_p50_seconds": 5.298, "tool_calls_per_task": 6.695652173913044, "timed_main_calls": 100, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 23, "passed": 17, "failed": 6, "errors": 0, "pass_rate": 73.91304347826087, "judge_mean": 4.565217391304348, "agent_gross_total": 4.329756575, "agent_gross_per_task": 0.18825028586956521, "judge_gross_total": 1.2946815, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 91.749, "turn_p95_seconds": 202.27114999999998, "task_p50_seconds": 161.557, "task_p95_seconds": 294.01589999999993, "tokens_per_task": 245427.13043478262, "tokens_per_turn": 166024.23529411765, "gross_cost_per_turn": 0.12734578161764706, "main_calls": 118, "main_call_p50_seconds": 18.823, "tool_calls_per_task": 7.043478260869565, "timed_main_calls": 118, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 23, "passed": 19, "failed": 4, "errors": 0, "pass_rate": 82.6086956521739, "judge_mean": 4.695652173913044, "agent_gross_total": 3.26055826, "agent_gross_per_task": 0.14176340260869563, "judge_gross_total": 1.1955645, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 66.337, "turn_p95_seconds": 194.08409999999998, "task_p50_seconds": 73.797, "task_p95_seconds": 318.9605999999999, "tokens_per_task": 213624.08695652173, "tokens_per_turn": 144510.41176470587, "gross_cost_per_turn": 0.09589877235294117, "main_calls": 106, "main_call_p50_seconds": 10.864, "tool_calls_per_task": 5.826086956521739, "timed_main_calls": 106, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 23, "passed": 18, "failed": 5, "errors": 0, "pass_rate": 78.26086956521739, "judge_mean": 4.478260869565218, "agent_gross_total": 3.919471745, "agent_gross_per_task": 0.170411815, "judge_gross_total": 1.295325, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 38.7085, "turn_p95_seconds": 129.72715, "task_p50_seconds": 50.307, "task_p95_seconds": 228.6620999999999, "tokens_per_task": 235397.30434782608, "tokens_per_turn": 159239.35294117648, "gross_cost_per_turn": 0.11527858073529412, "main_calls": 106, "main_call_p50_seconds": 6.691000000000001, "tool_calls_per_task": 6.869565217391305, "timed_main_calls": 106, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 23, "passed": 17, "failed": 6, "errors": 0, "pass_rate": 73.91304347826087, "judge_mean": 4.6521739130434785, "agent_gross_total": 3.67853044, "agent_gross_per_task": 0.15993610608695652, "judge_gross_total": 1.3434645, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 34.204, "turn_p95_seconds": 98.2451, "task_p50_seconds": 48.4, "task_p95_seconds": 123.1039, "tokens_per_task": 252786.60869565216, "tokens_per_turn": 171002.70588235295, "gross_cost_per_turn": 0.10819207176470588, "main_calls": 109, "main_call_p50_seconds": 5.845, "tool_calls_per_task": 8.26086956521739, "timed_main_calls": 109, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 23, "passed": 17, "failed": 6, "errors": 0, "pass_rate": 73.91304347826087, "judge_mean": 4.695652173913044, "agent_gross_total": 3.0493370349999998, "agent_gross_per_task": 0.1325798710869565, "judge_gross_total": 1.2841485, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 29.413, "turn_p95_seconds": 100.18559999999995, "task_p50_seconds": 42.757999999999996, "task_p95_seconds": 115.62669999999997, "tokens_per_task": 215987.78260869565, "tokens_per_turn": 146109.38235294117, "gross_cost_per_turn": 0.08968638338235294, "main_calls": 92, "main_call_p50_seconds": 4.04, "tool_calls_per_task": 6.173913043478261, "timed_main_calls": 92, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 23, "passed": 19, "failed": 4, "errors": 0, "pass_rate": 82.6086956521739, "judge_mean": 4.608695652173913, "agent_gross_total": 2.891633045, "agent_gross_per_task": 0.12572317586956522, "judge_gross_total": 1.2319305, "simulator_gross_total": 0, "turn_count": 34, "turn_p50_seconds": 34.314, "turn_p95_seconds": 87.34094999999999, "task_p50_seconds": 48.017, "task_p95_seconds": 129.54159999999996, "tokens_per_task": 195049.21739130435, "tokens_per_turn": 131945.0588235294, "gross_cost_per_turn": 0.08504803073529411, "main_calls": 85, "main_call_p50_seconds": 5.572, "tool_calls_per_task": 5.782608695652174, "timed_main_calls": 85, "native_main_timing_omitted_requests": 0 } } }, "suites": { "Catalog and shipping evidence": { "argon_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.500829, "agent_gross_per_task": 0.5002763333333333, "judge_gross_total": 0.2740905, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 120.1845, "turn_p95_seconds": 159.5846, "task_p50_seconds": 155.406, "task_p95_seconds": 236.2971, "tokens_per_task": 358630.6666666667, "tokens_per_turn": 268973.0, "gross_cost_per_turn": 0.37520725, "main_calls": 18, "main_call_p50_seconds": 11.883, "tool_calls_per_task": 15.666666666666666, "timed_main_calls": 18, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.99582295, "agent_gross_per_task": 0.6652743166666667, "judge_gross_total": 0.4990485, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 38.3715, "turn_p95_seconds": 175.18989999999994, "task_p50_seconds": 67.888, "task_p95_seconds": 185.97249999999997, "tokens_per_task": 885666.3333333334, "tokens_per_turn": 664249.75, "gross_cost_per_turn": 0.4989557375, "main_calls": 39, "main_call_p50_seconds": 3.848, "tool_calls_per_task": 33.333333333333336, "timed_main_calls": 39, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.91912448, "agent_gross_per_task": 0.30637482666666666, "judge_gross_total": 0.2142255, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 38.522, "turn_p95_seconds": 41.285199999999996, "task_p50_seconds": 41.662, "task_p95_seconds": 71.17209999999999, "tokens_per_task": 245507.0, "tokens_per_turn": 184130.25, "gross_cost_per_turn": 0.22978112, "main_calls": 11, "main_call_p50_seconds": 5.725, "tool_calls_per_task": 11.666666666666666, "timed_main_calls": 11, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.591690745, "agent_gross_per_task": 0.19723024833333333, "judge_gross_total": 0.1993935, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 78.3765, "turn_p95_seconds": 91.0036, "task_p50_seconds": 88.287, "task_p95_seconds": 123.5769, "tokens_per_task": 286589.0, "tokens_per_turn": 214941.75, "gross_cost_per_turn": 0.14792268625, "main_calls": 18, "main_call_p50_seconds": 7.851500000000001, "tool_calls_per_task": 11.0, "timed_main_calls": 18, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 3, "passed": 1, "failed": 2, "errors": 0, "pass_rate": 33.333333333333336, "judge_mean": 3.3333333333333335, "agent_gross_total": 0.45128282000000003, "agent_gross_per_task": 0.15042760666666669, "judge_gross_total": 0.212937, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 40.3985, "turn_p95_seconds": 65.84344999999999, "task_p50_seconds": 53.623, "task_p95_seconds": 66.5623, "tokens_per_task": 257469.33333333334, "tokens_per_turn": 193102.0, "gross_cost_per_turn": 0.11282070500000001, "main_calls": 14, "main_call_p50_seconds": 3.4555, "tool_calls_per_task": 10.0, "timed_main_calls": 14, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.439417775, "agent_gross_per_task": 0.14647259166666668, "judge_gross_total": 0.193233, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 45.474000000000004, "turn_p95_seconds": 52.32875, "task_p50_seconds": 52.625, "task_p95_seconds": 70.5926, "tokens_per_task": 231877.33333333334, "tokens_per_turn": 173908.0, "gross_cost_per_turn": 0.10985444375, "main_calls": 13, "main_call_p50_seconds": 4.65, "tool_calls_per_task": 9.333333333333334, "timed_main_calls": 13, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.0, "agent_gross_total": 0.657155875, "agent_gross_per_task": 0.21905195833333332, "judge_gross_total": 0.23745149999999998, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 59.66, "turn_p95_seconds": 158.17284999999995, "task_p50_seconds": 103.318, "task_p95_seconds": 167.2513, "tokens_per_task": 304283.0, "tokens_per_turn": 228212.25, "gross_cost_per_turn": 0.16428896875, "main_calls": 19, "main_call_p50_seconds": 11.723, "tool_calls_per_task": 11.0, "timed_main_calls": 19, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.0, "agent_gross_total": 0.438115155, "agent_gross_per_task": 0.146038385, "judge_gross_total": 0.18006, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 54.7655, "turn_p95_seconds": 74.51174999999999, "task_p50_seconds": 46.711, "task_p95_seconds": 130.1266, "tokens_per_task": 236221.0, "tokens_per_turn": 177165.75, "gross_cost_per_turn": 0.10952878875, "main_calls": 14, "main_call_p50_seconds": 7.899, "tool_calls_per_task": 7.0, "timed_main_calls": 14, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.0, "agent_gross_total": 0.66878825, "agent_gross_per_task": 0.22292941666666666, "judge_gross_total": 0.2261445, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 66.548, "turn_p95_seconds": 85.57845, "task_p50_seconds": 70.23, "task_p95_seconds": 143.0607, "tokens_per_task": 329399.6666666667, "tokens_per_turn": 247049.75, "gross_cost_per_turn": 0.1671970625, "main_calls": 22, "main_call_p50_seconds": 7.6765, "tool_calls_per_task": 14.333333333333334, "timed_main_calls": 22, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.0, "agent_gross_total": 0.4815159, "agent_gross_per_task": 0.1605053, "judge_gross_total": 0.2384205, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 40.512, "turn_p95_seconds": 59.51734999999999, "task_p50_seconds": 62.444, "task_p95_seconds": 70.1039, "tokens_per_task": 257800.66666666666, "tokens_per_turn": 193350.5, "gross_cost_per_turn": 0.120378975, "main_calls": 14, "main_call_p50_seconds": 5.4905, "tool_calls_per_task": 11.0, "timed_main_calls": 14, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 3.6666666666666665, "agent_gross_total": 0.35166295, "agent_gross_per_task": 0.11722098333333332, "judge_gross_total": 0.166545, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 28.294999999999998, "turn_p95_seconds": 37.4235, "task_p50_seconds": 38.169, "task_p95_seconds": 42.299099999999996, "tokens_per_task": 190484.33333333334, "tokens_per_turn": 142863.25, "gross_cost_per_turn": 0.0879157375, "main_calls": 11, "main_call_p50_seconds": 3.266, "tool_calls_per_task": 6.666666666666667, "timed_main_calls": 11, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 3, "passed": 2, "failed": 1, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.0, "agent_gross_total": 0.40036243, "agent_gross_per_task": 0.13345414333333333, "judge_gross_total": 0.16340100000000002, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 28.566499999999998, "turn_p95_seconds": 71.41094999999999, "task_p50_seconds": 43.11, "task_p95_seconds": 74.295, "tokens_per_task": 198830.0, "tokens_per_turn": 149122.5, "gross_cost_per_turn": 0.1000906075, "main_calls": 10, "main_call_p50_seconds": 3.5675, "tool_calls_per_task": 6.666666666666667, "timed_main_calls": 10, "native_main_timing_omitted_requests": 0 } }, "Support hours": { "argon_high": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.022091700000000002, "agent_gross_per_task": 0.022091700000000002, "judge_gross_total": 0.0113895, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 14.873, "turn_p95_seconds": 14.873, "task_p50_seconds": 14.873, "task_p95_seconds": 14.873, "tokens_per_task": 48615.0, "tokens_per_turn": 48615.0, "gross_cost_per_turn": 0.022091700000000002, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.021264999999999996, "agent_gross_per_task": 0.021264999999999996, "judge_gross_total": 0.013653, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 14.834, "turn_p95_seconds": 14.834, "task_p50_seconds": 14.834, "task_p95_seconds": 14.834, "tokens_per_task": 48661.0, "tokens_per_turn": 48661.0, "gross_cost_per_turn": 0.021264999999999996, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.020964399999999998, "agent_gross_per_task": 0.020964399999999998, "judge_gross_total": 0.012204, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 12.077, "turn_p95_seconds": 12.077, "task_p50_seconds": 12.077, "task_p95_seconds": 12.077, "tokens_per_task": 48172.0, "tokens_per_turn": 48172.0, "gross_cost_per_turn": 0.020964399999999998, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.022858499999999997, "agent_gross_per_task": 0.022858499999999997, "judge_gross_total": 0.0107385, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 16.107, "turn_p95_seconds": 16.107, "task_p50_seconds": 16.107, "task_p95_seconds": 16.107, "tokens_per_task": 49290.0, "tokens_per_turn": 49290.0, "gross_cost_per_turn": 0.022858499999999997, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.0193196, "agent_gross_per_task": 0.0193196, "judge_gross_total": 0.011637, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 12.502, "turn_p95_seconds": 12.502, "task_p50_seconds": 12.502, "task_p95_seconds": 12.502, "tokens_per_task": 47889.0, "tokens_per_turn": 47889.0, "gross_cost_per_turn": 0.0193196, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.0227528, "agent_gross_per_task": 0.0227528, "judge_gross_total": 0.01188, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 13.683, "turn_p95_seconds": 13.683, "task_p50_seconds": 13.683, "task_p95_seconds": 13.683, "tokens_per_task": 48884.0, "tokens_per_turn": 48884.0, "gross_cost_per_turn": 0.0227528, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.023676, "agent_gross_per_task": 0.023676, "judge_gross_total": 0.012246, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 16.062, "turn_p95_seconds": 16.062, "task_p50_seconds": 16.062, "task_p95_seconds": 16.062, "tokens_per_task": 49262.0, "tokens_per_turn": 49262.0, "gross_cost_per_turn": 0.023676, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.020628599999999997, "agent_gross_per_task": 0.020628599999999997, "judge_gross_total": 0.012594, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 14.892, "turn_p95_seconds": 14.892, "task_p50_seconds": 14.892, "task_p95_seconds": 14.892, "tokens_per_task": 48391.0, "tokens_per_turn": 48391.0, "gross_cost_per_turn": 0.020628599999999997, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.0200093, "agent_gross_per_task": 0.0200093, "judge_gross_total": 0.012153, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 11.646, "turn_p95_seconds": 11.646, "task_p50_seconds": 11.646, "task_p95_seconds": 11.646, "tokens_per_task": 47787.0, "tokens_per_turn": 47787.0, "gross_cost_per_turn": 0.0200093, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.0216503, "agent_gross_per_task": 0.0216503, "judge_gross_total": 0.011757, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 13.479, "turn_p95_seconds": 13.479, "task_p50_seconds": 13.479, "task_p95_seconds": 13.479, "tokens_per_task": 48443.0, "tokens_per_turn": 48443.0, "gross_cost_per_turn": 0.0216503, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.020275599999999998, "agent_gross_per_task": 0.020275599999999998, "judge_gross_total": 0.0112905, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 12.945, "turn_p95_seconds": 12.945, "task_p50_seconds": 12.945, "task_p95_seconds": 12.945, "tokens_per_task": 47916.0, "tokens_per_turn": 47916.0, "gross_cost_per_turn": 0.020275599999999998, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 1, "passed": 1, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.01643958, "agent_gross_per_task": 0.01643958, "judge_gross_total": 0.0120975, "simulator_gross_total": 0, "turn_count": 1, "turn_p50_seconds": 14.138, "turn_p95_seconds": 14.138, "task_p50_seconds": 14.138, "task_p95_seconds": 14.138, "tokens_per_task": 48118.0, "tokens_per_turn": 48118.0, "gross_cost_per_turn": 0.01643958, "main_calls": 0, "main_call_p50_seconds": null, "tool_calls_per_task": 0.0, "timed_main_calls": 0, "native_main_timing_omitted_requests": 0 } }, "Purchase assistance": { "argon_high": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 2.9498007, "agent_gross_per_task": 0.49163344999999997, "judge_gross_total": 0.7819665, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 72.858, "turn_p95_seconds": 95.27409999999999, "task_p50_seconds": 79.3845, "task_p95_seconds": 113.37174999999999, "tokens_per_task": 306063.6666666667, "tokens_per_turn": 262340.28571428574, "gross_cost_per_turn": 0.4214001, "main_calls": 28, "main_call_p50_seconds": 7.1215, "tool_calls_per_task": 11.833333333333334, "timed_main_calls": 28, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 5.995584, "agent_gross_per_task": 0.999264, "judge_gross_total": 3.3217485, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 46.868, "turn_p95_seconds": 332.11659999999995, "task_p50_seconds": 54.8575, "task_p95_seconds": 336.1665, "tokens_per_task": 1326014.5, "tokens_per_turn": 1136583.857142857, "gross_cost_per_turn": 0.856512, "main_calls": 61, "main_call_p50_seconds": 4.171, "tool_calls_per_task": 29.833333333333332, "timed_main_calls": 61, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 6, "passed": 6, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 3.881797345, "agent_gross_per_task": 0.6469662241666666, "judge_gross_total": 1.6468995, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 41.464, "turn_p95_seconds": 124.95609999999995, "task_p50_seconds": 55.97, "task_p95_seconds": 128.18124999999998, "tokens_per_task": 477467.1666666667, "tokens_per_turn": 409257.5714285714, "gross_cost_per_turn": 0.5545424778571428, "main_calls": 31, "main_call_p50_seconds": 4.204, "tool_calls_per_task": 13.666666666666666, "timed_main_calls": 31, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.4861887, "agent_gross_per_task": 0.24769811666666666, "judge_gross_total": 1.272891, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 90.706, "turn_p95_seconds": 168.40759999999997, "task_p50_seconds": 112.887, "task_p95_seconds": 173.15925, "tokens_per_task": 428812.5, "tokens_per_turn": 367553.5714285714, "gross_cost_per_turn": 0.21231267142857144, "main_calls": 36, "main_call_p50_seconds": 10.1085, "tool_calls_per_task": 15.166666666666666, "timed_main_calls": 36, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.053004815, "agent_gross_per_task": 0.1755008025, "judge_gross_total": 1.217463, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 56.199, "turn_p95_seconds": 91.22549999999998, "task_p50_seconds": 69.1605, "task_p95_seconds": 91.86574999999999, "tokens_per_task": 375615.5, "tokens_per_turn": 321956.14285714284, "gross_cost_per_turn": 0.1504292592857143, "main_calls": 32, "main_call_p50_seconds": 3.551, "tool_calls_per_task": 12.5, "timed_main_calls": 32, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.135518125, "agent_gross_per_task": 0.1892530208333333, "judge_gross_total": 1.1872095, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 55.336, "turn_p95_seconds": 119.56189999999997, "task_p50_seconds": 67.889, "task_p95_seconds": 121.98325, "tokens_per_task": 357912.3333333333, "tokens_per_turn": 306782.0, "gross_cost_per_turn": 0.16221687499999998, "main_calls": 27, "main_call_p50_seconds": 4.801, "tool_calls_per_task": 12.333333333333334, "timed_main_calls": 27, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 6, "passed": 3, "failed": 3, "errors": 0, "pass_rate": 50.0, "judge_mean": 3.8333333333333335, "agent_gross_total": 1.436470135, "agent_gross_per_task": 0.23941168916666666, "judge_gross_total": 1.247229, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 115.025, "turn_p95_seconds": 205.88939999999997, "task_p50_seconds": 152.248, "task_p95_seconds": 210.88775, "tokens_per_task": 383327.0, "tokens_per_turn": 328566.0, "gross_cost_per_turn": 0.20521001928571428, "main_calls": 30, "main_call_p50_seconds": 12.197, "tool_calls_per_task": 13.5, "timed_main_calls": 30, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.06679706, "agent_gross_per_task": 0.17779951000000002, "judge_gross_total": 0.8919225, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 83.156, "turn_p95_seconds": 148.7601, "task_p50_seconds": 86.5705, "task_p95_seconds": 201.97275000000002, "tokens_per_task": 312752.8333333333, "tokens_per_turn": 268073.85714285716, "gross_cost_per_turn": 0.15239958, "main_calls": 27, "main_call_p50_seconds": 11.916, "tool_calls_per_task": 9.666666666666666, "timed_main_calls": 27, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.484984725, "agent_gross_per_task": 0.24749745416666666, "judge_gross_total": 1.387737, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 51.951, "turn_p95_seconds": 129.06529999999998, "task_p50_seconds": 93.17500000000001, "task_p95_seconds": 129.22775, "tokens_per_task": 414664.1666666667, "tokens_per_turn": 355426.4285714286, "gross_cost_per_turn": 0.21214067499999997, "main_calls": 31, "main_call_p50_seconds": 5.846, "tool_calls_per_task": 12.166666666666666, "timed_main_calls": 31, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 6, "passed": 4, "failed": 2, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.666666666666667, "agent_gross_total": 1.317797725, "agent_gross_per_task": 0.21963295416666664, "judge_gross_total": 1.085736, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 41.206, "turn_p95_seconds": 180.87939999999992, "task_p50_seconds": 78.786, "task_p95_seconds": 186.58749999999998, "tokens_per_task": 379729.0, "tokens_per_turn": 325482.0, "gross_cost_per_turn": 0.18825681785714285, "main_calls": 27, "main_call_p50_seconds": 7.52, "tool_calls_per_task": 15.0, "timed_main_calls": 27, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 6, "passed": 4, "failed": 2, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.5, "agent_gross_total": 0.9851374, "agent_gross_per_task": 0.16418956666666668, "judge_gross_total": 0.924786, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 54.673, "turn_p95_seconds": 111.88419999999999, "task_p50_seconds": 73.407, "task_p95_seconds": 112.907, "tokens_per_task": 284581.1666666667, "tokens_per_turn": 243926.7142857143, "gross_cost_per_turn": 0.1407339142857143, "main_calls": 21, "main_call_p50_seconds": 5.291, "tool_calls_per_task": 8.666666666666666, "timed_main_calls": 21, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 6, "passed": 5, "failed": 1, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 0.8635198749999999, "agent_gross_per_task": 0.14391997916666666, "judge_gross_total": 0.9258105, "simulator_gross_total": 0, "turn_count": 7, "turn_p50_seconds": 55.327, "turn_p95_seconds": 79.47569999999999, "task_p50_seconds": 62.302, "task_p95_seconds": 106.66324999999999, "tokens_per_task": 273784.0, "tokens_per_turn": 234672.0, "gross_cost_per_turn": 0.12335998214285714, "main_calls": 20, "main_call_p50_seconds": 9.7615, "tool_calls_per_task": 10.333333333333334, "timed_main_calls": 20, "native_main_timing_omitted_requests": 0 } }, "Product measurements": { "argon_high": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.103624425, "agent_gross_per_task": 0.27590610625, "judge_gross_total": 0.1743765, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 36.4945, "turn_p95_seconds": 127.26269999999997, "task_p50_seconds": 36.4945, "task_p95_seconds": 127.26269999999997, "tokens_per_task": 162816.5, "tokens_per_turn": 162816.5, "gross_cost_per_turn": 0.27590610625, "main_calls": 12, "main_call_p50_seconds": 6.703, "tool_calls_per_task": 8.75, "timed_main_calls": 11, "native_main_timing_omitted_requests": 1 }, "argon_low": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.70988965, "agent_gross_per_task": 0.1774724125, "judge_gross_total": 0.24519, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 28.683500000000002, "turn_p95_seconds": 36.57015, "task_p50_seconds": 28.683500000000002, "task_p95_seconds": 36.57015, "tokens_per_task": 156889.75, "tokens_per_turn": 156889.75, "gross_cost_per_turn": 0.1774724125, "main_calls": 10, "main_call_p50_seconds": 3.839, "tool_calls_per_task": 7.75, "timed_main_calls": 10, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.68544047, "agent_gross_per_task": 0.1713601175, "judge_gross_total": 0.146757, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 25.1125, "turn_p95_seconds": 39.77514999999999, "task_p50_seconds": 25.1125, "task_p95_seconds": 39.77514999999999, "tokens_per_task": 128897.25, "tokens_per_turn": 128897.25, "gross_cost_per_turn": 0.1713601175, "main_calls": 9, "main_call_p50_seconds": 4.298, "tool_calls_per_task": 6.5, "timed_main_calls": 9, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.561887475, "agent_gross_per_task": 0.14047186875, "judge_gross_total": 0.168693, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 52.32, "turn_p95_seconds": 108.44519999999997, "task_p50_seconds": 52.32, "task_p95_seconds": 108.44519999999997, "tokens_per_task": 182546.25, "tokens_per_turn": 182546.25, "gross_cost_per_turn": 0.14047186875, "main_calls": 15, "main_call_p50_seconds": 6.737, "tool_calls_per_task": 8.75, "timed_main_calls": 15, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.27818295, "agent_gross_per_task": 0.0695457375, "judge_gross_total": 0.121332, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 25.8645, "turn_p95_seconds": 35.81305, "task_p50_seconds": 25.8645, "task_p95_seconds": 35.81305, "tokens_per_task": 117711.5, "tokens_per_turn": 117711.5, "gross_cost_per_turn": 0.0695457375, "main_calls": 10, "main_call_p50_seconds": 3.2915, "tool_calls_per_task": 5.25, "timed_main_calls": 10, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.36636005, "agent_gross_per_task": 0.0915900125, "judge_gross_total": 0.203946, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 31.622, "turn_p95_seconds": 51.945699999999995, "task_p50_seconds": 31.622, "task_p95_seconds": 51.945699999999995, "tokens_per_task": 151097.0, "tokens_per_turn": 151097.0, "gross_cost_per_turn": 0.0915900125, "main_calls": 12, "main_call_p50_seconds": 4.6265, "tool_calls_per_task": 7.25, "timed_main_calls": 12, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 4, "passed": 3, "failed": 1, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.25, "agent_gross_total": 0.39784495, "agent_gross_per_task": 0.0994612375, "judge_gross_total": 0.1974735, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 72.941, "turn_p95_seconds": 119.4519, "task_p50_seconds": 72.941, "task_p95_seconds": 119.4519, "tokens_per_task": 159241.75, "tokens_per_turn": 159241.75, "gross_cost_per_turn": 0.0994612375, "main_calls": 12, "main_call_p50_seconds": 15.667000000000002, "tool_calls_per_task": 7.25, "timed_main_calls": 12, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 4, "passed": 3, "failed": 1, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.25, "agent_gross_total": 0.374987275, "agent_gross_per_task": 0.09374681875, "judge_gross_total": 0.1466055, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 34.285000000000004, "turn_p95_seconds": 67.06519999999999, "task_p50_seconds": 34.285000000000004, "task_p95_seconds": 67.06519999999999, "tokens_per_task": 145562.0, "tokens_per_turn": 145562.0, "gross_cost_per_turn": 0.09374681875, "main_calls": 12, "main_call_p50_seconds": 5.462, "tool_calls_per_task": 7.0, "timed_main_calls": 12, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 4, "passed": 3, "failed": 1, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.25, "agent_gross_total": 0.3378449, "agent_gross_per_task": 0.084461225, "judge_gross_total": 0.127989, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 30.290999999999997, "turn_p95_seconds": 61.63029999999999, "task_p50_seconds": 30.290999999999997, "task_p95_seconds": 61.63029999999999, "tokens_per_task": 161198.25, "tokens_per_turn": 161198.25, "gross_cost_per_turn": 0.084461225, "main_calls": 12, "main_call_p50_seconds": 3.9015, "tool_calls_per_task": 6.75, "timed_main_calls": 12, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.39399509, "agent_gross_per_task": 0.0984987725, "judge_gross_total": 0.1507755, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 27.718, "turn_p95_seconds": 63.47844999999998, "task_p50_seconds": 27.718, "task_p95_seconds": 63.47844999999998, "tokens_per_task": 149372.5, "tokens_per_turn": 149372.5, "gross_cost_per_turn": 0.0984987725, "main_calls": 13, "main_call_p50_seconds": 4.394, "tool_calls_per_task": 6.25, "timed_main_calls": 13, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.37466794999999997, "agent_gross_per_task": 0.09366698749999999, "judge_gross_total": 0.2605725, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 29.941499999999998, "turn_p95_seconds": 45.21849999999999, "task_p50_seconds": 29.941499999999998, "task_p95_seconds": 45.21849999999999, "tokens_per_task": 148999.5, "tokens_per_turn": 148999.5, "gross_cost_per_turn": 0.09366698749999999, "main_calls": 10, "main_call_p50_seconds": 4.055, "tool_calls_per_task": 7.5, "timed_main_calls": 10, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 4, "passed": 4, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.30231600000000003, "agent_gross_per_task": 0.07557900000000001, "judge_gross_total": 0.1328205, "simulator_gross_total": 0, "turn_count": 4, "turn_p50_seconds": 28.1785, "turn_p95_seconds": 32.7178, "task_p50_seconds": 28.1785, "task_p95_seconds": 32.7178, "tokens_per_task": 116113.75, "tokens_per_turn": 116113.75, "gross_cost_per_turn": 0.07557900000000001, "main_calls": 9, "main_call_p50_seconds": 4.642, "tool_calls_per_task": 5.5, "timed_main_calls": 9, "native_main_timing_omitted_requests": 0 } }, "Customer reply quality": { "argon_high": { "observations": 12, "passed": 7, "failed": 5, "errors": 0, "pass_rate": 58.333333333333336, "judge_mean": 4.416666666666667, "agent_gross_total": 3.8442630749999998, "agent_gross_per_task": 0.32035525625, "judge_gross_total": 0.465813, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 45.155, "turn_p95_seconds": 95.64229999999996, "task_p50_seconds": 47.625, "task_p95_seconds": 118.65289999999999, "tokens_per_task": 187778.08333333334, "tokens_per_turn": 150222.46666666667, "gross_cost_per_turn": 0.256284205, "main_calls": 44, "main_call_p50_seconds": 8.428, "tool_calls_per_task": 5.833333333333333, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "argon_low": { "observations": 12, "passed": 10, "failed": 2, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 5.0, "agent_gross_total": 2.33535584, "agent_gross_per_task": 0.19461298666666668, "judge_gross_total": 0.330327, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 24.693, "turn_p95_seconds": 41.82639999999999, "task_p50_seconds": 29.23, "task_p95_seconds": 59.89839999999998, "tokens_per_task": 161956.16666666666, "tokens_per_turn": 129564.93333333333, "gross_cost_per_turn": 0.15569038933333335, "main_calls": 41, "main_call_p50_seconds": 3.914, "tool_calls_per_task": 4.75, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 12, "passed": 10, "failed": 2, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 5.0, "agent_gross_total": 2.81906879, "agent_gross_per_task": 0.23492239916666668, "judge_gross_total": 0.4963965, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 24.154, "turn_p95_seconds": 44.01219999999999, "task_p50_seconds": 27.265, "task_p95_seconds": 67.85684999999998, "tokens_per_task": 190776.08333333334, "tokens_per_turn": 152620.86666666667, "gross_cost_per_turn": 0.18793791933333334, "main_calls": 45, "main_call_p50_seconds": 3.554, "tool_calls_per_task": 6.166666666666667, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 12, "passed": 8, "failed": 4, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.083333333333333, "agent_gross_total": 2.0936694350000002, "agent_gross_per_task": 0.1744724529166667, "judge_gross_total": 0.450294, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 68.35, "turn_p95_seconds": 164.04489999999996, "task_p50_seconds": 91.855, "task_p95_seconds": 169.54734999999997, "tokens_per_task": 210277.5, "tokens_per_turn": 168222.0, "gross_cost_per_turn": 0.13957796233333336, "main_calls": 57, "main_call_p50_seconds": 9.478, "tool_calls_per_task": 6.416666666666667, "timed_main_calls": 57, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 12, "passed": 9, "failed": 3, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.666666666666667, "agent_gross_total": 1.181946035, "agent_gross_per_task": 0.09849550291666666, "judge_gross_total": 0.43044899999999997, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 28.88, "turn_p95_seconds": 66.0934, "task_p50_seconds": 29.177, "task_p95_seconds": 82.27004999999998, "tokens_per_task": 160055.33333333334, "tokens_per_turn": 128044.26666666666, "gross_cost_per_turn": 0.07879640233333333, "main_calls": 41, "main_call_p50_seconds": 3.509, "tool_calls_per_task": 4.166666666666667, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 12, "passed": 9, "failed": 3, "errors": 0, "pass_rate": 75.0, "judge_mean": 5.0, "agent_gross_total": 1.40726094, "agent_gross_per_task": 0.117271745, "judge_gross_total": 0.405453, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 32.489, "turn_p95_seconds": 74.37769999999999, "task_p50_seconds": 39.2735, "task_p95_seconds": 82.29939999999999, "tokens_per_task": 169638.83333333334, "tokens_per_turn": 135711.06666666668, "gross_cost_per_turn": 0.093817396, "main_calls": 45, "main_call_p50_seconds": 5.551, "tool_calls_per_task": 4.333333333333333, "timed_main_calls": 45, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 12, "passed": 10, "failed": 2, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.75, "agent_gross_total": 2.0092849999999998, "agent_gross_per_task": 0.16744041666666665, "judge_gross_total": 0.4627035, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 95.089, "turn_p95_seconds": 254.37739999999985, "task_p50_seconds": 125.15849999999999, "task_p95_seconds": 346.59694999999977, "tokens_per_task": 209953.5, "tokens_per_turn": 167962.8, "gross_cost_per_turn": 0.1339523333333333, "main_calls": 58, "main_call_p50_seconds": 22.4315, "tool_calls_per_task": 5.583333333333333, "timed_main_calls": 58, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 12, "passed": 9, "failed": 3, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.583333333333333, "agent_gross_total": 1.303483835, "agent_gross_per_task": 0.10862365291666666, "judge_gross_total": 0.418212, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 62.927, "turn_p95_seconds": 245.32069999999993, "task_p50_seconds": 108.57300000000001, "task_p95_seconds": 254.01754999999997, "tokens_per_task": 161720.16666666666, "tokens_per_turn": 129376.13333333333, "gross_cost_per_turn": 0.08689892233333334, "main_calls": 44, "main_call_p50_seconds": 21.431, "tool_calls_per_task": 3.75, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 12, "passed": 10, "failed": 2, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.75, "agent_gross_total": 1.6213496, "agent_gross_per_task": 0.13511246666666668, "judge_gross_total": 0.448884, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 46.513, "turn_p95_seconds": 92.77269999999992, "task_p50_seconds": 54.3185, "task_p95_seconds": 123.62064999999996, "tokens_per_task": 179533.5, "tokens_per_turn": 143626.8, "gross_cost_per_turn": 0.10808997333333334, "main_calls": 50, "main_call_p50_seconds": 8.536999999999999, "tool_calls_per_task": 5.0, "timed_main_calls": 50, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 12, "passed": 9, "failed": 3, "errors": 0, "pass_rate": 75.0, "judge_mean": 4.75, "agent_gross_total": 1.40866768, "agent_gross_per_task": 0.11738897333333333, "judge_gross_total": 0.3804405, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 32.726, "turn_p95_seconds": 63.483899999999984, "task_p50_seconds": 42.326, "task_p95_seconds": 74.61185, "tokens_per_task": 177659.08333333334, "tokens_per_turn": 142127.26666666666, "gross_cost_per_turn": 0.09391117866666666, "main_calls": 46, "main_call_p50_seconds": 5.3575, "tool_calls_per_task": 6.0, "timed_main_calls": 46, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 12, "passed": 8, "failed": 4, "errors": 0, "pass_rate": 66.66666666666667, "judge_mean": 4.666666666666667, "agent_gross_total": 1.259561145, "agent_gross_per_task": 0.10496342874999999, "judge_gross_total": 0.4017285, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 28.48, "turn_p95_seconds": 79.11109999999994, "task_p50_seconds": 29.914, "task_p95_seconds": 107.78544999999998, "tokens_per_task": 168625.0, "tokens_per_turn": 134900.0, "gross_cost_per_turn": 0.083970743, "main_calls": 43, "main_call_p50_seconds": 4.265, "tool_calls_per_task": 4.0, "timed_main_calls": 43, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 12, "passed": 10, "failed": 2, "errors": 0, "pass_rate": 83.33333333333333, "judge_mean": 4.666666666666667, "agent_gross_total": 1.18376097, "agent_gross_per_task": 0.0986467475, "judge_gross_total": 0.349998, "simulator_gross_total": 0, "turn_count": 15, "turn_p50_seconds": 29.389, "turn_p95_seconds": 136.41639999999998, "task_p50_seconds": 42.962, "task_p95_seconds": 146.10189999999997, "tokens_per_task": 148414.16666666666, "tokens_per_turn": 118731.33333333333, "gross_cost_per_turn": 0.078917398, "main_calls": 39, "main_call_p50_seconds": 8.713, "tool_calls_per_task": 3.6666666666666665, "timed_main_calls": 39, "native_main_timing_omitted_requests": 0 } }, "Return policy and confirmation": { "argon_high": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 4.0744033, "agent_gross_per_task": 0.5820576142857143, "judge_gross_total": 0.452301, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 47.542, "turn_p95_seconds": 127.98845, "task_p50_seconds": 117.999, "task_p95_seconds": 247.35329999999993, "tokens_per_task": 380225.0, "tokens_per_turn": 190112.5, "gross_cost_per_turn": 0.29102880714285717, "main_calls": 52, "main_call_p50_seconds": 6.693, "tool_calls_per_task": 9.142857142857142, "timed_main_calls": 51, "native_main_timing_omitted_requests": 1 }, "argon_low": { "observations": 7, "passed": 6, "failed": 1, "errors": 0, "pass_rate": 85.71428571428571, "judge_mean": 4.571428571428571, "agent_gross_total": 3.65415763, "agent_gross_per_task": 0.5220225185714286, "judge_gross_total": 0.4615065, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 30.1485, "turn_p95_seconds": 127.12994999999998, "task_p50_seconds": 55.195, "task_p95_seconds": 239.02169999999987, "tokens_per_task": 469697.85714285716, "tokens_per_turn": 234848.92857142858, "gross_cost_per_turn": 0.2610112592857143, "main_calls": 73, "main_call_p50_seconds": 3.629, "tool_calls_per_task": 12.428571428571429, "timed_main_calls": 73, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 3.8383594149999998, "agent_gross_per_task": 0.5483370592857143, "judge_gross_total": 0.43886400000000003, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 31.601, "turn_p95_seconds": 100.05149999999999, "task_p50_seconds": 60.765, "task_p95_seconds": 179.5343999999999, "tokens_per_task": 350236.0, "tokens_per_turn": 175118.0, "gross_cost_per_turn": 0.27416852964285715, "main_calls": 48, "main_call_p50_seconds": 3.6645, "tool_calls_per_task": 8.857142857142858, "timed_main_calls": 48, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 7, "passed": 6, "failed": 1, "errors": 0, "pass_rate": 85.71428571428571, "judge_mean": 5.0, "agent_gross_total": 1.77030794, "agent_gross_per_task": 0.25290113428571426, "judge_gross_total": 0.3665505, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 62.4475, "turn_p95_seconds": 115.888, "task_p50_seconds": 136.532, "task_p95_seconds": 281.81179999999995, "tokens_per_task": 309232.0, "tokens_per_turn": 154616.0, "gross_cost_per_turn": 0.12645056714285713, "main_calls": 44, "main_call_p50_seconds": 9.722, "tool_calls_per_task": 6.714285714285714, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.224672615, "agent_gross_per_task": 0.17495323071428573, "judge_gross_total": 0.3141525, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 28.436, "turn_p95_seconds": 89.4313, "task_p50_seconds": 56.872, "task_p95_seconds": 170.4700999999999, "tokens_per_task": 264663.85714285716, "tokens_per_turn": 132331.92857142858, "gross_cost_per_turn": 0.08747661535714286, "main_calls": 37, "main_call_p50_seconds": 3.387, "tool_calls_per_task": 5.142857142857143, "timed_main_calls": 37, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.375676125, "agent_gross_per_task": 0.19652516071428572, "judge_gross_total": 0.32925150000000003, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 37.6695, "turn_p95_seconds": 103.5325, "task_p50_seconds": 81.34100000000001, "task_p95_seconds": 195.47539999999992, "tokens_per_task": 280203.5714285714, "tokens_per_turn": 140101.7857142857, "gross_cost_per_turn": 0.09826258035714286, "main_calls": 40, "main_call_p50_seconds": 7.125, "tool_calls_per_task": 5.714285714285714, "timed_main_calls": 40, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 7, "passed": 6, "failed": 1, "errors": 0, "pass_rate": 85.71428571428571, "judge_mean": 5.0, "agent_gross_total": 1.6041774899999999, "agent_gross_per_task": 0.22916821285714284, "judge_gross_total": 0.317223, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 85.3965, "turn_p95_seconds": 176.06969999999998, "task_p50_seconds": 193.907, "task_p95_seconds": 287.2197, "tokens_per_task": 271812.28571428574, "tokens_per_turn": 135906.14285714287, "gross_cost_per_turn": 0.11458410642857142, "main_calls": 38, "main_call_p50_seconds": 23.814999999999998, "tool_calls_per_task": 5.571428571428571, "timed_main_calls": 38, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.3161822349999999, "agent_gross_per_task": 0.18802603357142855, "judge_gross_total": 0.2983935, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 79.49000000000001, "turn_p95_seconds": 194.0841, "task_p50_seconds": 245.282, "task_p95_seconds": 334.68919999999997, "tokens_per_task": 264882.28571428574, "tokens_per_turn": 132441.14285714287, "gross_cost_per_turn": 0.09401301678571428, "main_calls": 40, "main_call_p50_seconds": 15.061, "tool_calls_per_task": 5.428571428571429, "timed_main_calls": 40, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 7, "passed": 6, "failed": 1, "errors": 0, "pass_rate": 85.71428571428571, "judge_mean": 4.428571428571429, "agent_gross_total": 1.4965807899999999, "agent_gross_per_task": 0.2137972557142857, "judge_gross_total": 0.291612, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 41.3065, "turn_p95_seconds": 141.4926, "task_p50_seconds": 78.578, "task_p95_seconds": 244.53609999999998, "tokens_per_task": 262528.71428571426, "tokens_per_turn": 131264.35714285713, "gross_cost_per_turn": 0.10689862785714285, "main_calls": 37, "main_call_p50_seconds": 6.392, "tool_calls_per_task": 4.857142857142857, "timed_main_calls": 37, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.5115053, "agent_gross_per_task": 0.21592932857142857, "judge_gross_total": 0.356067, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 34.204, "turn_p95_seconds": 108.03774999999999, "task_p50_seconds": 86.009, "task_p95_seconds": 211.57569999999993, "tokens_per_task": 313591.0, "tokens_per_turn": 156795.5, "gross_cost_per_turn": 0.10796466428571429, "main_calls": 44, "main_call_p50_seconds": 6.4094999999999995, "tool_calls_per_task": 6.714285714285714, "timed_main_calls": 44, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.33867689, "agent_gross_per_task": 0.1912395557142857, "judge_gross_total": 0.392514, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 31.7625, "turn_p95_seconds": 100.1856, "task_p50_seconds": 63.525, "task_p95_seconds": 187.6079999999999, "tokens_per_task": 301411.28571428574, "tokens_per_turn": 150705.64285714287, "gross_cost_per_turn": 0.09561977785714285, "main_calls": 41, "main_call_p50_seconds": 3.937, "tool_calls_per_task": 6.571428571428571, "timed_main_calls": 41, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 7, "passed": 7, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 1.147624595, "agent_gross_per_task": 0.1639463707142857, "judge_gross_total": 0.3010275, "simulator_gross_total": 0, "turn_count": 14, "turn_p50_seconds": 36.3555, "turn_p95_seconds": 87.34094999999999, "task_p50_seconds": 85.117, "task_p95_seconds": 166.79389999999992, "tokens_per_task": 255813.85714285713, "tokens_per_turn": 127906.92857142857, "gross_cost_per_turn": 0.08197318535714285, "main_calls": 36, "main_call_p50_seconds": 4.758, "tool_calls_per_task": 5.0, "timed_main_calls": 36, "native_main_timing_omitted_requests": 0 } }, "Conversation ownership": { "argon_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.8886816500000001, "agent_gross_per_task": 0.2962272166666667, "judge_gross_total": 0.251322, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 84.513, "turn_p95_seconds": 131.1861, "task_p50_seconds": 84.513, "task_p95_seconds": 131.1861, "tokens_per_task": 174404.66666666666, "tokens_per_turn": 174404.66666666666, "gross_cost_per_turn": 0.2962272166666667, "main_calls": 6, "main_call_p50_seconds": 8.4, "tool_calls_per_task": 10.0, "timed_main_calls": 5, "native_main_timing_omitted_requests": 1 }, "argon_low": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.473817735, "agent_gross_per_task": 0.157939245, "judge_gross_total": 0.2438025, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 28.971, "turn_p95_seconds": 54.59309999999999, "task_p50_seconds": 28.971, "task_p95_seconds": 54.59309999999999, "tokens_per_task": 176533.0, "tokens_per_turn": 176533.0, "gross_cost_per_turn": 0.157939245, "main_calls": 6, "main_call_p50_seconds": 3.8415, "tool_calls_per_task": 9.0, "timed_main_calls": 6, "native_main_timing_omitted_requests": 0 }, "argon_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.4786422, "agent_gross_per_task": 0.1595474, "judge_gross_total": 0.166224, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 29.404, "turn_p95_seconds": 35.284600000000005, "task_p50_seconds": 29.404, "task_p95_seconds": 35.284600000000005, "tokens_per_task": 125900.66666666667, "tokens_per_turn": 125900.66666666667, "gross_cost_per_turn": 0.1595474, "main_calls": 5, "main_call_p50_seconds": 3.718, "tool_calls_per_task": 6.0, "timed_main_calls": 5, "native_main_timing_omitted_requests": 0 }, "baseline_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.28881361, "agent_gross_per_task": 0.09627120333333333, "judge_gross_total": 0.221652, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 48.891, "turn_p95_seconds": 64.26570000000001, "task_p50_seconds": 48.891, "task_p95_seconds": 64.26570000000001, "tokens_per_task": 166666.33333333334, "tokens_per_turn": 166666.33333333334, "gross_cost_per_turn": 0.09627120333333333, "main_calls": 8, "main_call_p50_seconds": 9.873999999999999, "tool_calls_per_task": 7.333333333333333, "timed_main_calls": 8, "native_main_timing_omitted_requests": 0 }, "baseline_low": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.217415885, "agent_gross_per_task": 0.07247196166666667, "judge_gross_total": 0.173691, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 27.489, "turn_p95_seconds": 44.58, "task_p50_seconds": 27.489, "task_p95_seconds": 44.58, "tokens_per_task": 134295.0, "tokens_per_turn": 134295.0, "gross_cost_per_turn": 0.07247196166666667, "main_calls": 7, "main_call_p50_seconds": 3.689, "tool_calls_per_task": 5.333333333333333, "timed_main_calls": 7, "native_main_timing_omitted_requests": 0 }, "baseline_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.22610254999999999, "agent_gross_per_task": 0.07536751666666666, "judge_gross_total": 0.173787, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 36.812, "turn_p95_seconds": 37.0766, "task_p50_seconds": 36.812, "task_p95_seconds": 37.0766, "tokens_per_task": 121795.66666666667, "tokens_per_turn": 121795.66666666667, "gross_cost_per_turn": 0.07536751666666666, "main_calls": 5, "main_call_p50_seconds": 3.727, "tool_calls_per_task": 5.666666666666667, "timed_main_calls": 5, "native_main_timing_omitted_requests": 0 }, "flash35_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.37882188499999997, "agent_gross_per_task": 0.12627396166666666, "judge_gross_total": 0.13896, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 111.885, "turn_p95_seconds": 187.70999999999998, "task_p50_seconds": 111.885, "task_p95_seconds": 187.70999999999998, "tokens_per_task": 151041.66666666666, "tokens_per_turn": 151041.66666666666, "gross_cost_per_turn": 0.12627396166666666, "main_calls": 9, "main_call_p50_seconds": 18.307, "tool_calls_per_task": 5.333333333333333, "timed_main_calls": 9, "native_main_timing_omitted_requests": 0 }, "flash35_low": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.20407018500000002, "agent_gross_per_task": 0.068023395, "judge_gross_total": 0.1508355, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 34.952, "turn_p95_seconds": 36.8429, "task_p50_seconds": 34.952, "task_p95_seconds": 36.8429, "tokens_per_task": 124530.0, "tokens_per_turn": 124530.0, "gross_cost_per_turn": 0.068023395, "main_calls": 7, "main_call_p50_seconds": 3.61, "tool_calls_per_task": 4.666666666666667, "timed_main_calls": 7, "native_main_timing_omitted_requests": 0 }, "flash35_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.34261918, "agent_gross_per_task": 0.11420639333333334, "judge_gross_total": 0.224919, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 32.326, "turn_p95_seconds": 65.1364, "task_p50_seconds": 32.326, "task_p95_seconds": 65.1364, "tokens_per_task": 166783.66666666666, "tokens_per_turn": 166783.66666666666, "gross_cost_per_turn": 0.11420639333333334, "main_calls": 8, "main_call_p50_seconds": 5.7059999999999995, "tool_calls_per_task": 7.0, "timed_main_calls": 8, "native_main_timing_omitted_requests": 0 }, "flash37_high": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.24923995000000002, "agent_gross_per_task": 0.08307998333333334, "judge_gross_total": 0.20092649999999998, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 48.4, "turn_p95_seconds": 72.4399, "task_p50_seconds": 48.4, "task_p95_seconds": 72.4399, "tokens_per_task": 150063.33333333334, "tokens_per_turn": 150063.33333333334, "gross_cost_per_turn": 0.08307998333333334, "main_calls": 6, "main_call_p50_seconds": 7.353, "tool_calls_per_task": 7.666666666666667, "timed_main_calls": 6, "native_main_timing_omitted_requests": 0 }, "flash37_low": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.16258197, "agent_gross_per_task": 0.05419399, "judge_gross_total": 0.137706, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 26.697, "turn_p95_seconds": 58.224, "task_p50_seconds": 26.697, "task_p95_seconds": 58.224, "tokens_per_task": 111680.0, "tokens_per_turn": 111680.0, "gross_cost_per_turn": 0.05419399, "main_calls": 5, "main_call_p50_seconds": 5.049, "tool_calls_per_task": 5.666666666666667, "timed_main_calls": 5, "native_main_timing_omitted_requests": 0 }, "flash37_medium": { "observations": 3, "passed": 3, "failed": 0, "errors": 0, "pass_rate": 100.0, "judge_mean": 5.0, "agent_gross_total": 0.252548, "agent_gross_per_task": 0.08418266666666667, "judge_gross_total": 0.1673085, "simulator_gross_total": 0, "turn_count": 3, "turn_p50_seconds": 35.63, "turn_p95_seconds": 46.7783, "task_p50_seconds": 35.63, "task_p95_seconds": 46.7783, "tokens_per_task": 113636.33333333333, "tokens_per_turn": 113636.33333333333, "gross_cost_per_turn": 0.08418266666666667, "main_calls": 5, "main_call_p50_seconds": 6.992, "tool_calls_per_task": 4.666666666666667, "timed_main_calls": 5, "native_main_timing_omitted_requests": 0 } } }, "observations_by_task": [ { "task": "task-10", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [54.212], "agent_gross_standard_cost_usd": 0.2455771, "judge_gross_standard_cost_usd": 0.0375765, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 207310, "agent_prompt_tokens": 199351, "agent_cached_input_tokens": 81856, "agent_thinking_tokens": 6235, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 83543, "cached_input_tokens": 0, "completion_tokens": 1264, "thoughts_tokens": 3649, "total_tokens": 88456, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 115126, "cached_input_tokens": 81856, "completion_tokens": 407, "thoughts_tokens": 2386, "total_tokens": 117919, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 53, "thoughts_tokens": 200, "total_tokens": 935, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 5, "main_agent_timed_model_calls": 5 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.839], "agent_gross_standard_cost_usd": 0.09758069999999999, "judge_gross_standard_cost_usd": 0.021999, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 135645, "agent_prompt_tokens": 129304, "agent_cached_input_tokens": 25732, "agent_thinking_tokens": 4909, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77158, "cached_input_tokens": 0, "completion_tokens": 1116, "thoughts_tokens": 2205, "total_tokens": 80479, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 52146, "cached_input_tokens": 25732, "completion_tokens": 316, "thoughts_tokens": 2704, "total_tokens": 55166, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [144.307], "agent_gross_standard_cost_usd": 1.9346169, "judge_gross_standard_cost_usd": 0.882765, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 1400403, "agent_prompt_tokens": 1392213, "agent_cached_input_tokens": 591243, "agent_thinking_tokens": 3708, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 225871, "cached_input_tokens": 0, "completion_tokens": 1121, "thoughts_tokens": 3708, "total_tokens": 230700, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 936827, "cached_input_tokens": 515733, "completion_tokens": 1180, "thoughts_tokens": 0, "total_tokens": 938007, "request_count": 10 }, "gemini-3.1-flash-lite": { "prompt_tokens": 229515, "cached_input_tokens": 75510, "completion_tokens": 2181, "thoughts_tokens": 0, "total_tokens": 231696, "request_count": 14 } }, "agent_requests": 26, "main_agent_model_calls": 10, "tool_calls": 28, "main_agent_timed_model_calls": 10 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [134.09], "agent_gross_standard_cost_usd": 0.297472475, "judge_gross_standard_cost_usd": 0.43425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 629083, "agent_prompt_tokens": 612668, "agent_cached_input_tokens": 241400, "agent_thinking_tokens": 12651, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 153187, "cached_input_tokens": 0, "completion_tokens": 1006, "thoughts_tokens": 4203, "total_tokens": 158396, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 186780, "cached_input_tokens": 109155, "completion_tokens": 597, "thoughts_tokens": 7812, "total_tokens": 195189, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 272701, "cached_input_tokens": 132245, "completion_tokens": 2161, "thoughts_tokens": 636, "total_tokens": 275498, "request_count": 18 } }, "agent_requests": 25, "main_agent_model_calls": 5, "tool_calls": 21, "main_agent_timed_model_calls": 5 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [79.804], "agent_gross_standard_cost_usd": 0.53280785, "judge_gross_standard_cost_usd": 0.4246125, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 472513, "agent_prompt_tokens": 461843, "agent_cached_input_tokens": 131074, "agent_thinking_tokens": 7083, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 155535, "cached_input_tokens": 0, "completion_tokens": 1226, "thoughts_tokens": 3699, "total_tokens": 160460, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 121418, "cached_input_tokens": 33094, "completion_tokens": 1687, "thoughts_tokens": 0, "total_tokens": 123105, "request_count": 10 }, "gemini-4-argon": { "prompt_tokens": 184890, "cached_input_tokens": 97980, "completion_tokens": 674, "thoughts_tokens": 3384, "total_tokens": 188948, "request_count": 3 } }, "agent_requests": 15, "main_agent_model_calls": 3, "tool_calls": 13, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [71.722], "agent_gross_standard_cost_usd": 0.209554125, "judge_gross_standard_cost_usd": 0.2858445, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 453363, "agent_prompt_tokens": 444205, "agent_cached_input_tokens": 105243, "agent_thinking_tokens": 5358, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 126266, "cached_input_tokens": 0, "completion_tokens": 974, "thoughts_tokens": 3119, "total_tokens": 130359, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 196814, "cached_input_tokens": 44275, "completion_tokens": 2528, "thoughts_tokens": 0, "total_tokens": 199342, "request_count": 15 }, "gemini-3.8-flash": { "prompt_tokens": 121125, "cached_input_tokens": 60968, "completion_tokens": 298, "thoughts_tokens": 2239, "total_tokens": 123662, "request_count": 4 } }, "agent_requests": 21, "main_agent_model_calls": 4, "tool_calls": 14, "main_agent_timed_model_calls": 4 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [34.969, 22.759], "agent_gross_standard_cost_usd": 0.480191245, "judge_gross_standard_cost_usd": 0.104826, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 317584, "agent_prompt_tokens": 308234, "agent_cached_input_tokens": 75601, "agent_thinking_tokens": 5582, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 127816, "cached_input_tokens": 10134, "completion_tokens": 2565, "thoughts_tokens": 4808, "total_tokens": 135189, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 36427, "cached_input_tokens": 15973, "completion_tokens": 367, "thoughts_tokens": 0, "total_tokens": 36794, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 143991, "cached_input_tokens": 49494, "completion_tokens": 836, "thoughts_tokens": 774, "total_tokens": 145601, "request_count": 5 } }, "agent_requests": 14, "main_agent_model_calls": 5, "tool_calls": 16, "main_agent_timed_model_calls": 5 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [38.618, 25.438], "agent_gross_standard_cost_usd": 0.197000875, "judge_gross_standard_cost_usd": 0.099711, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 310808, "agent_prompt_tokens": 300925, "agent_cached_input_tokens": 90147, "agent_thinking_tokens": 7048, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 126944, "cached_input_tokens": 0, "completion_tokens": 1906, "thoughts_tokens": 3891, "total_tokens": 132741, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25506, "cached_input_tokens": 7045, "completion_tokens": 256, "thoughts_tokens": 0, "total_tokens": 25762, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 148475, "cached_input_tokens": 83102, "completion_tokens": 673, "thoughts_tokens": 3157, "total_tokens": 152305, "request_count": 6 } }, "agent_requests": 14, "main_agent_model_calls": 6, "tool_calls": 13, "main_agent_timed_model_calls": 6 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.358], "agent_gross_standard_cost_usd": 0.3049083, "judge_gross_standard_cost_usd": 0.061422, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 167679, "agent_prompt_tokens": 163963, "agent_cached_input_tokens": 18286, "agent_thinking_tokens": 2115, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62227, "cached_input_tokens": 0, "completion_tokens": 935, "thoughts_tokens": 2115, "total_tokens": 65277, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 18148, "cached_input_tokens": 0, "completion_tokens": 222, "thoughts_tokens": 0, "total_tokens": 18370, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 83588, "cached_input_tokens": 18286, "completion_tokens": 444, "thoughts_tokens": 0, "total_tokens": 84032, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [55.336], "agent_gross_standard_cost_usd": 0.1232525, "judge_gross_standard_cost_usd": 0.0578175, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 179078, "agent_prompt_tokens": 170572, "agent_cached_input_tokens": 61032, "agent_thinking_tokens": 7066, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61494, "cached_input_tokens": 0, "completion_tokens": 926, "thoughts_tokens": 3591, "total_tokens": 66011, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 13062, "cached_input_tokens": 0, "completion_tokens": 217, "thoughts_tokens": 0, "total_tokens": 13279, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 96016, "cached_input_tokens": 61032, "completion_tokens": 297, "thoughts_tokens": 3475, "total_tokens": 99788, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 7, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.702], "agent_gross_standard_cost_usd": 0.1298247, "judge_gross_standard_cost_usd": 0.015177, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 117516, "agent_prompt_tokens": 114276, "agent_cached_input_tokens": 37905, "agent_thinking_tokens": 2193, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52844, "cached_input_tokens": 0, "completion_tokens": 808, "thoughts_tokens": 2193, "total_tokens": 55845, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 61432, "cached_input_tokens": 37905, "completion_tokens": 239, "thoughts_tokens": 0, "total_tokens": 61671, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.489], "agent_gross_standard_cost_usd": 0.07694147, "judge_gross_standard_cost_usd": 0.0129045, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109764, "agent_prompt_tokens": 104471, "agent_cached_input_tokens": 38015, "agent_thinking_tokens": 4367, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52261, "cached_input_tokens": 10149, "completion_tokens": 723, "thoughts_tokens": 2555, "total_tokens": 55539, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 52210, "cached_input_tokens": 27866, "completion_tokens": 203, "thoughts_tokens": 1812, "total_tokens": 54225, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.047], "agent_gross_standard_cost_usd": 0.1521702, "judge_gross_standard_cost_usd": 0.022467, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109781, "agent_prompt_tokens": 105141, "agent_cached_input_tokens": 20115, "agent_thinking_tokens": 3053, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59124, "cached_input_tokens": 0, "completion_tokens": 1130, "thoughts_tokens": 2439, "total_tokens": 62693, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 45359, "cached_input_tokens": 20115, "completion_tokens": 399, "thoughts_tokens": 614, "total_tokens": 46372, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [53.489], "agent_gross_standard_cost_usd": 0.11687354999999999, "judge_gross_standard_cost_usd": 0.022497, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 153156, "agent_prompt_tokens": 144524, "agent_cached_input_tokens": 52993, "agent_thinking_tokens": 6949, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59372, "cached_input_tokens": 0, "completion_tokens": 1019, "thoughts_tokens": 3120, "total_tokens": 63511, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 84494, "cached_input_tokens": 52993, "completion_tokens": 606, "thoughts_tokens": 3829, "total_tokens": 88929, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [22.553], "agent_gross_standard_cost_usd": 0.1422912, "judge_gross_standard_cost_usd": 0.0390795, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 105595, "agent_prompt_tokens": 101695, "agent_cached_input_tokens": 18497, "agent_thinking_tokens": 2237, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56591, "cached_input_tokens": 0, "completion_tokens": 1153, "thoughts_tokens": 2237, "total_tokens": 59981, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 44450, "cached_input_tokens": 18497, "completion_tokens": 454, "thoughts_tokens": 0, "total_tokens": 44904, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [44.219], "agent_gross_standard_cost_usd": 0.0934185, "judge_gross_standard_cost_usd": 0.040185, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 125903, "agent_prompt_tokens": 118747, "agent_cached_input_tokens": 37826, "agent_thinking_tokens": 5715, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56307, "cached_input_tokens": 0, "completion_tokens": 990, "thoughts_tokens": 2727, "total_tokens": 60024, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 61786, "cached_input_tokens": 37826, "completion_tokens": 395, "thoughts_tokens": 2988, "total_tokens": 65169, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.548], "agent_gross_standard_cost_usd": 0.2005433, "judge_gross_standard_cost_usd": 0.015993, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 117322, "agent_prompt_tokens": 113758, "agent_cached_input_tokens": 18602, "agent_thinking_tokens": 2334, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52378, "cached_input_tokens": 0, "completion_tokens": 911, "thoughts_tokens": 2334, "total_tokens": 55623, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 952, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 1002, "request_count": 1 }, "gemini-4-argon": { "prompt_tokens": 60428, "cached_input_tokens": 18602, "completion_tokens": 269, "thoughts_tokens": 0, "total_tokens": 60697, "request_count": 3 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [38.498], "agent_gross_standard_cost_usd": 0.08750735, "judge_gross_standard_cost_usd": 0.0156435, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 126274, "agent_prompt_tokens": 119848, "agent_cached_input_tokens": 41435, "agent_thinking_tokens": 5276, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52402, "cached_input_tokens": 0, "completion_tokens": 924, "thoughts_tokens": 3404, "total_tokens": 56730, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 67446, "cached_input_tokens": 41435, "completion_tokens": 226, "thoughts_tokens": 1872, "total_tokens": 69544, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.912], "agent_gross_standard_cost_usd": 0.1766161, "judge_gross_standard_cost_usd": 0.0316425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 144705, "agent_prompt_tokens": 140968, "agent_cached_input_tokens": 47924, "agent_thinking_tokens": 2489, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59431, "cached_input_tokens": 0, "completion_tokens": 939, "thoughts_tokens": 2489, "total_tokens": 62859, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 81537, "cached_input_tokens": 47924, "completion_tokens": 309, "thoughts_tokens": 0, "total_tokens": 81846, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.307], "agent_gross_standard_cost_usd": 0.09369995, "judge_gross_standard_cost_usd": 0.0278775, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 133438, "agent_prompt_tokens": 127815, "agent_cached_input_tokens": 41435, "agent_thinking_tokens": 4553, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59244, "cached_input_tokens": 0, "completion_tokens": 816, "thoughts_tokens": 1817, "total_tokens": 61877, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 68571, "cached_input_tokens": 41435, "completion_tokens": 254, "thoughts_tokens": 2736, "total_tokens": 71561, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.848], "agent_gross_standard_cost_usd": 0.3111786, "judge_gross_standard_cost_usd": 0.0455055, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 237287, "agent_prompt_tokens": 232690, "agent_cached_input_tokens": 79936, "agent_thinking_tokens": 2178, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 64393, "cached_input_tokens": 0, "completion_tokens": 1453, "thoughts_tokens": 2021, "total_tokens": 67867, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 141451, "cached_input_tokens": 79936, "completion_tokens": 577, "thoughts_tokens": 0, "total_tokens": 142028, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 26174, "cached_input_tokens": 0, "completion_tokens": 343, "thoughts_tokens": 0, "total_tokens": 26517, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 46, "thoughts_tokens": 157, "total_tokens": 875, "request_count": 1 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [78.64], "agent_gross_standard_cost_usd": 0.15164105, "judge_gross_standard_cost_usd": 0.0383085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 148716, "agent_prompt_tokens": 135439, "agent_cached_input_tokens": 40541, "agent_thinking_tokens": 11907, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62933, "cached_input_tokens": 0, "completion_tokens": 1106, "thoughts_tokens": 3063, "total_tokens": 67102, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 72506, "cached_input_tokens": 40541, "completion_tokens": 264, "thoughts_tokens": 8844, "total_tokens": 81614, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.948], "agent_gross_standard_cost_usd": 0.2851749, "judge_gross_standard_cost_usd": 0.0279015, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 165089, "agent_prompt_tokens": 159990, "agent_cached_input_tokens": 43723, "agent_thinking_tokens": 3351, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59646, "cached_input_tokens": 0, "completion_tokens": 1152, "thoughts_tokens": 2407, "total_tokens": 63205, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 98848, "cached_input_tokens": 43723, "completion_tokens": 486, "thoughts_tokens": 944, "total_tokens": 100278, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 7, "main_agent_timed_model_calls": 4 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.627], "agent_gross_standard_cost_usd": 0.09800579999999999, "judge_gross_standard_cost_usd": 0.0331785, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 137707, "agent_prompt_tokens": 131245, "agent_cached_input_tokens": 35774, "agent_thinking_tokens": 5007, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63389, "cached_input_tokens": 0, "completion_tokens": 1020, "thoughts_tokens": 3067, "total_tokens": 67476, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 66360, "cached_input_tokens": 35774, "completion_tokens": 325, "thoughts_tokens": 1940, "total_tokens": 68625, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.304, 24.154, 22.127], "agent_gross_standard_cost_usd": 0.51395323, "judge_gross_standard_cost_usd": 0.0427845, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 424434, "agent_prompt_tokens": 413418, "agent_cached_input_tokens": 144652, "agent_thinking_tokens": 6671, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 183978, "cached_input_tokens": 10131, "completion_tokens": 3262, "thoughts_tokens": 5584, "total_tokens": 192824, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1988, "cached_input_tokens": 0, "completion_tokens": 170, "thoughts_tokens": 0, "total_tokens": 2158, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 227452, "cached_input_tokens": 134521, "completion_tokens": 913, "thoughts_tokens": 1087, "total_tokens": 229452, "request_count": 9 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 10, "main_agent_timed_model_calls": 9 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.33, 21.208, 20.649], "agent_gross_standard_cost_usd": 0.19617128, "judge_gross_standard_cost_usd": 0.026235, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 370173, "agent_prompt_tokens": 358199, "agent_cached_input_tokens": 135179, "agent_thinking_tokens": 8175, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 182289, "cached_input_tokens": 16331, "completion_tokens": 2956, "thoughts_tokens": 5757, "total_tokens": 191002, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 2269, "cached_input_tokens": 0, "completion_tokens": 155, "thoughts_tokens": 0, "total_tokens": 2424, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 173641, "cached_input_tokens": 118848, "completion_tokens": 688, "thoughts_tokens": 2418, "total_tokens": 176747, "request_count": 9 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 7, "main_agent_timed_model_calls": 9 }, { "task": "task-16", "suite": "Product measurements", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [21.935], "agent_gross_standard_cost_usd": 0.1528787, "judge_gross_standard_cost_usd": 0.037158, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122149, "agent_prompt_tokens": 118772, "agent_cached_input_tokens": 23591, "agent_thinking_tokens": 2018, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58416, "cached_input_tokens": 0, "completion_tokens": 869, "thoughts_tokens": 2018, "total_tokens": 61303, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12912, "cached_input_tokens": 5116, "completion_tokens": 179, "thoughts_tokens": 0, "total_tokens": 13091, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 47444, "cached_input_tokens": 18475, "completion_tokens": 311, "thoughts_tokens": 0, "total_tokens": 47755, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-16", "suite": "Product measurements", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.535], "agent_gross_standard_cost_usd": 0.08356195, "judge_gross_standard_cost_usd": 0.0402465, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140941, "agent_prompt_tokens": 137125, "agent_cached_input_tokens": 41351, "agent_thinking_tokens": 2298, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57756, "cached_input_tokens": 0, "completion_tokens": 1055, "thoughts_tokens": 1394, "total_tokens": 60205, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14520, "cached_input_tokens": 5316, "completion_tokens": 200, "thoughts_tokens": 0, "total_tokens": 14720, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 64849, "cached_input_tokens": 36035, "completion_tokens": 263, "thoughts_tokens": 904, "total_tokens": 66016, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.349], "agent_gross_standard_cost_usd": 0.21716884, "judge_gross_standard_cost_usd": 0.0472785, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 164395, "agent_prompt_tokens": 159176, "agent_cached_input_tokens": 57558, "agent_thinking_tokens": 3504, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61186, "cached_input_tokens": 10148, "completion_tokens": 979, "thoughts_tokens": 1514, "total_tokens": 63679, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17064, "cached_input_tokens": 0, "completion_tokens": 379, "thoughts_tokens": 0, "total_tokens": 17443, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 80926, "cached_input_tokens": 47410, "completion_tokens": 357, "thoughts_tokens": 1990, "total_tokens": 83273, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [54.811], "agent_gross_standard_cost_usd": 0.108824175, "judge_gross_standard_cost_usd": 0.0358065, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 146411, "agent_prompt_tokens": 137557, "agent_cached_input_tokens": 47463, "agent_thinking_tokens": 7355, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56692, "cached_input_tokens": 0, "completion_tokens": 927, "thoughts_tokens": 2956, "total_tokens": 60575, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16406, "cached_input_tokens": 8757, "completion_tokens": 290, "thoughts_tokens": 0, "total_tokens": 16696, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 64459, "cached_input_tokens": 38706, "completion_tokens": 282, "thoughts_tokens": 4399, "total_tokens": 69140, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-14", "suite": "Product measurements", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.035], "agent_gross_standard_cost_usd": 0.15378438, "judge_gross_standard_cost_usd": 0.0248715, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 106319, "agent_prompt_tokens": 102327, "agent_cached_input_tokens": 34934, "agent_thinking_tokens": 2954, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56346, "cached_input_tokens": 16326, "completion_tokens": 841, "thoughts_tokens": 2117, "total_tokens": 59304, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 45981, "cached_input_tokens": 18608, "completion_tokens": 197, "thoughts_tokens": 837, "total_tokens": 47015, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-14", "suite": "Product measurements", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.709], "agent_gross_standard_cost_usd": 0.10161047499999999, "judge_gross_standard_cost_usd": 0.097182, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 193577, "agent_prompt_tokens": 188855, "agent_cached_input_tokens": 59965, "agent_thinking_tokens": 3284, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67037, "cached_input_tokens": 0, "completion_tokens": 794, "thoughts_tokens": 1701, "total_tokens": 69532, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 72046, "cached_input_tokens": 40692, "completion_tokens": 197, "thoughts_tokens": 1583, "total_tokens": 73826, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 49772, "cached_input_tokens": 19273, "completion_tokens": 447, "thoughts_tokens": 0, "total_tokens": 50219, "request_count": 5 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.19], "agent_gross_standard_cost_usd": 0.16160855, "judge_gross_standard_cost_usd": 0.037449, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122726, "agent_prompt_tokens": 118953, "agent_cached_input_tokens": 23608, "agent_thinking_tokens": 2263, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58515, "cached_input_tokens": 0, "completion_tokens": 934, "thoughts_tokens": 1922, "total_tokens": 61371, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12909, "cached_input_tokens": 5116, "completion_tokens": 181, "thoughts_tokens": 0, "total_tokens": 13090, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 47529, "cached_input_tokens": 18492, "completion_tokens": 395, "thoughts_tokens": 341, "total_tokens": 48265, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.515], "agent_gross_standard_cost_usd": 0.07236345, "judge_gross_standard_cost_usd": 0.030711, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 123459, "agent_prompt_tokens": 118713, "agent_cached_input_tokens": 37438, "agent_thinking_tokens": 3315, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51560, "cached_input_tokens": 0, "completion_tokens": 990, "thoughts_tokens": 2109, "total_tokens": 54659, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14529, "cached_input_tokens": 5316, "completion_tokens": 202, "thoughts_tokens": 0, "total_tokens": 14731, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 52624, "cached_input_tokens": 32122, "completion_tokens": 239, "thoughts_tokens": 1206, "total_tokens": 54069, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.894], "agent_gross_standard_cost_usd": 0.19447495, "judge_gross_standard_cost_usd": 0.039078, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 127554, "agent_prompt_tokens": 121392, "agent_cached_input_tokens": 19924, "agent_thinking_tokens": 4736, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57703, "cached_input_tokens": 0, "completion_tokens": 971, "thoughts_tokens": 2848, "total_tokens": 61522, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 48772, "cached_input_tokens": 19924, "completion_tokens": 328, "thoughts_tokens": 1888, "total_tokens": 50988, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14917, "cached_input_tokens": 0, "completion_tokens": 127, "thoughts_tokens": 0, "total_tokens": 15044, "request_count": 2 } }, "agent_requests": 6, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [50.65], "agent_gross_standard_cost_usd": 0.11869905, "judge_gross_standard_cost_usd": 0.0568005, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 166446, "agent_prompt_tokens": 158691, "agent_cached_input_tokens": 47754, "agent_thinking_tokens": 6149, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60904, "cached_input_tokens": 0, "completion_tokens": 943, "thoughts_tokens": 1970, "total_tokens": 63817, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 68384, "cached_input_tokens": 35502, "completion_tokens": 261, "thoughts_tokens": 4179, "total_tokens": 72824, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29403, "cached_input_tokens": 12252, "completion_tokens": 402, "thoughts_tokens": 0, "total_tokens": 29805, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.662], "agent_gross_standard_cost_usd": 0.24898915, "judge_gross_standard_cost_usd": 0.066987, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 195213, "agent_prompt_tokens": 188976, "agent_cached_input_tokens": 53342, "agent_thinking_tokens": 4640, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67050, "cached_input_tokens": 0, "completion_tokens": 848, "thoughts_tokens": 2807, "total_tokens": 70705, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 92571, "cached_input_tokens": 53342, "completion_tokens": 380, "thoughts_tokens": 1833, "total_tokens": 94784, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29355, "cached_input_tokens": 0, "completion_tokens": 369, "thoughts_tokens": 0, "total_tokens": 29724, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 12, "main_agent_timed_model_calls": 3 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [52.625], "agent_gross_standard_cost_usd": 0.121459675, "judge_gross_standard_cost_usd": 0.0550395, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 185384, "agent_prompt_tokens": 177832, "agent_cached_input_tokens": 69074, "agent_thinking_tokens": 5915, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61099, "cached_input_tokens": 0, "completion_tokens": 994, "thoughts_tokens": 2478, "total_tokens": 64571, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 96177, "cached_input_tokens": 60271, "completion_tokens": 275, "thoughts_tokens": 3437, "total_tokens": 99889, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20556, "cached_input_tokens": 8803, "completion_tokens": 368, "thoughts_tokens": 0, "total_tokens": 20924, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 8, "main_agent_timed_model_calls": 4 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.301, 39.15], "agent_gross_standard_cost_usd": 0.47566038, "judge_gross_standard_cost_usd": 0.1081605, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 413754, "agent_prompt_tokens": 403213, "agent_cached_input_tokens": 153914, "agent_thinking_tokens": 7169, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 145059, "cached_input_tokens": 10136, "completion_tokens": 1954, "thoughts_tokens": 5403, "total_tokens": 152416, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 192691, "cached_input_tokens": 109304, "completion_tokens": 812, "thoughts_tokens": 1766, "total_tokens": 195269, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 65463, "cached_input_tokens": 34474, "completion_tokens": 606, "thoughts_tokens": 0, "total_tokens": 66069, "request_count": 7 } }, "agent_requests": 17, "main_agent_model_calls": 6, "tool_calls": 16, "main_agent_timed_model_calls": 6 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.291, 40.298], "agent_gross_standard_cost_usd": 0.19925905, "judge_gross_standard_cost_usd": 0.081393, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 343802, "agent_prompt_tokens": 333083, "agent_cached_input_tokens": 106030, "agent_thinking_tokens": 7817, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 134091, "cached_input_tokens": 0, "completion_tokens": 1913, "thoughts_tokens": 4575, "total_tokens": 140579, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 141384, "cached_input_tokens": 78656, "completion_tokens": 480, "thoughts_tokens": 3242, "total_tokens": 145106, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 57608, "cached_input_tokens": 27374, "completion_tokens": 509, "thoughts_tokens": 0, "total_tokens": 58117, "request_count": 6 } }, "agent_requests": 16, "main_agent_model_calls": 6, "tool_calls": 12, "main_agent_timed_model_calls": 6 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [31.365], "agent_gross_standard_cost_usd": 0.30749519999999997, "judge_gross_standard_cost_usd": 0.033477, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 171899, "agent_prompt_tokens": 167624, "agent_cached_input_tokens": 46442, "agent_thinking_tokens": 2978, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59751, "cached_input_tokens": 0, "completion_tokens": 992, "thoughts_tokens": 1919, "total_tokens": 62662, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 107873, "cached_input_tokens": 46442, "completion_tokens": 305, "thoughts_tokens": 1059, "total_tokens": 109237, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.232], "agent_gross_standard_cost_usd": 0.0996252, "judge_gross_standard_cost_usd": 0.033882, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 159664, "agent_prompt_tokens": 154530, "agent_cached_input_tokens": 62056, "agent_thinking_tokens": 4195, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59716, "cached_input_tokens": 0, "completion_tokens": 702, "thoughts_tokens": 2346, "total_tokens": 62764, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 94814, "cached_input_tokens": 62056, "completion_tokens": 237, "thoughts_tokens": 1849, "total_tokens": 96900, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.457], "agent_gross_standard_cost_usd": 0.1780183, "judge_gross_standard_cost_usd": 0.0291615, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 117637, "agent_prompt_tokens": 112038, "agent_cached_input_tokens": 35527, "agent_thinking_tokens": 4361, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52853, "cached_input_tokens": 0, "completion_tokens": 929, "thoughts_tokens": 2017, "total_tokens": 55799, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 59185, "cached_input_tokens": 35527, "completion_tokens": 309, "thoughts_tokens": 2344, "total_tokens": 61838, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [45.725], "agent_gross_standard_cost_usd": 0.1029114, "judge_gross_standard_cost_usd": 0.028797, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 134601, "agent_prompt_tokens": 127767, "agent_cached_input_tokens": 43746, "agent_thinking_tokens": 5763, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52760, "cached_input_tokens": 0, "completion_tokens": 853, "thoughts_tokens": 2672, "total_tokens": 56285, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 75007, "cached_input_tokens": 43746, "completion_tokens": 218, "thoughts_tokens": 3091, "total_tokens": 78316, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.661], "agent_gross_standard_cost_usd": 0.27995760000000003, "judge_gross_standard_cost_usd": 0.0317205, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155965, "agent_prompt_tokens": 151309, "agent_cached_input_tokens": 40960, "agent_thinking_tokens": 3514, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55292, "cached_input_tokens": 0, "completion_tokens": 907, "thoughts_tokens": 2417, "total_tokens": 58616, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 96017, "cached_input_tokens": 40960, "completion_tokens": 235, "thoughts_tokens": 1097, "total_tokens": 97349, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.246], "agent_gross_standard_cost_usd": 0.09309035, "judge_gross_standard_cost_usd": 0.032037, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 153940, "agent_prompt_tokens": 148785, "agent_cached_input_tokens": 64951, "agent_thinking_tokens": 3819, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55209, "cached_input_tokens": 0, "completion_tokens": 1106, "thoughts_tokens": 1857, "total_tokens": 58172, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 93576, "cached_input_tokens": 64951, "completion_tokens": 230, "thoughts_tokens": 1962, "total_tokens": 95768, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.071, 31.837, 23.186], "agent_gross_standard_cost_usd": 0.64552225, "judge_gross_standard_cost_usd": 0.085032, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 528362, "agent_prompt_tokens": 516808, "agent_cached_input_tokens": 192112, "agent_thinking_tokens": 6361, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 211536, "cached_input_tokens": 10135, "completion_tokens": 3937, "thoughts_tokens": 5561, "total_tokens": 221034, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 303855, "cached_input_tokens": 181977, "completion_tokens": 1157, "thoughts_tokens": 411, "total_tokens": 305423, "request_count": 10 }, "gemini-3.8-flash": { "prompt_tokens": 1417, "cached_input_tokens": 0, "completion_tokens": 99, "thoughts_tokens": 389, "total_tokens": 1905, "request_count": 2 } }, "agent_requests": 18, "main_agent_model_calls": 10, "tool_calls": 11, "main_agent_timed_model_calls": 10 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [51.912, 30.568, 23.576], "agent_gross_standard_cost_usd": 0.25222814, "judge_gross_standard_cost_usd": 0.041478, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 356783, "agent_prompt_tokens": 341005, "agent_cached_input_tokens": 81572, "agent_thinking_tokens": 11023, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 192560, "cached_input_tokens": 10148, "completion_tokens": 3629, "thoughts_tokens": 5847, "total_tokens": 202036, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 148445, "cached_input_tokens": 71424, "completion_tokens": 1126, "thoughts_tokens": 5176, "total_tokens": 154747, "request_count": 9 } }, "agent_requests": 15, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.464], "agent_gross_standard_cost_usd": 0.38369595, "judge_gross_standard_cost_usd": 0.1356975, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 299314, "agent_prompt_tokens": 294778, "agent_cached_input_tokens": 76140, "agent_thinking_tokens": 2481, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 99599, "cached_input_tokens": 0, "completion_tokens": 1175, "thoughts_tokens": 2481, "total_tokens": 103255, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 153036, "cached_input_tokens": 76140, "completion_tokens": 541, "thoughts_tokens": 0, "total_tokens": 153577, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 42143, "cached_input_tokens": 0, "completion_tokens": 339, "thoughts_tokens": 0, "total_tokens": 42482, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [85.663], "agent_gross_standard_cost_usd": 0.21065745, "judge_gross_standard_cost_usd": 0.2875875, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 439497, "agent_prompt_tokens": 429568, "agent_cached_input_tokens": 153130, "agent_thinking_tokens": 7116, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 123809, "cached_input_tokens": 0, "completion_tokens": 1129, "thoughts_tokens": 2577, "total_tokens": 127515, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 159006, "cached_input_tokens": 102894, "completion_tokens": 449, "thoughts_tokens": 4539, "total_tokens": 163994, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 146753, "cached_input_tokens": 50236, "completion_tokens": 1235, "thoughts_tokens": 0, "total_tokens": 147988, "request_count": 11 } }, "agent_requests": 18, "main_agent_model_calls": 5, "tool_calls": 15, "main_agent_timed_model_calls": 5 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.938], "agent_gross_standard_cost_usd": 0.28062235, "judge_gross_standard_cost_usd": 0.082959, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 179024, "agent_prompt_tokens": 174577, "agent_cached_input_tokens": 15600, "agent_thinking_tokens": 1985, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66667, "cached_input_tokens": 0, "completion_tokens": 1496, "thoughts_tokens": 1363, "total_tokens": 69526, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 52089, "cached_input_tokens": 15600, "completion_tokens": 599, "thoughts_tokens": 0, "total_tokens": 52688, "request_count": 7 }, "gemini-4-argon": { "prompt_tokens": 55821, "cached_input_tokens": 0, "completion_tokens": 367, "thoughts_tokens": 622, "total_tokens": 56810, "request_count": 2 } }, "agent_requests": 11, "main_agent_model_calls": 2, "tool_calls": 11, "main_agent_timed_model_calls": 2 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.106], "agent_gross_standard_cost_usd": 0.111337375, "judge_gross_standard_cost_usd": 0.087333, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 174634, "agent_prompt_tokens": 167594, "agent_cached_input_tokens": 31634, "agent_thinking_tokens": 4526, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67513, "cached_input_tokens": 0, "completion_tokens": 1406, "thoughts_tokens": 2526, "total_tokens": 71445, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 48740, "cached_input_tokens": 14297, "completion_tokens": 308, "thoughts_tokens": 2000, "total_tokens": 51048, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 51341, "cached_input_tokens": 17337, "completion_tokens": 800, "thoughts_tokens": 0, "total_tokens": 52141, "request_count": 6 } }, "agent_requests": 10, "main_agent_model_calls": 2, "tool_calls": 10, "main_agent_timed_model_calls": 2 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.37], "agent_gross_standard_cost_usd": 0.0076508999999999995, "judge_gross_standard_cost_usd": 0.009084, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16659, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 733, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 473, "thoughts_tokens": 733, "total_tokens": 16659, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [5.224], "agent_gross_standard_cost_usd": 0.0083159, "judge_gross_standard_cost_usd": 0.009237, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16925, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 1006, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 466, "thoughts_tokens": 1006, "total_tokens": 16925, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [34.638, 22.448, 21.739], "agent_gross_standard_cost_usd": 0.7660368, "judge_gross_standard_cost_usd": 0.079977, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 447403, "agent_prompt_tokens": 435449, "agent_cached_input_tokens": 70427, "agent_thinking_tokens": 7570, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 210268, "cached_input_tokens": 0, "completion_tokens": 3421, "thoughts_tokens": 6023, "total_tokens": 219712, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 224504, "cached_input_tokens": 70427, "completion_tokens": 925, "thoughts_tokens": 1404, "total_tokens": 226833, "request_count": 8 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 38, "thoughts_tokens": 143, "total_tokens": 858, "request_count": 1 } }, "agent_requests": 15, "main_agent_model_calls": 8, "tool_calls": 8, "main_agent_timed_model_calls": 8 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.39, 16.201, 31.745], "agent_gross_standard_cost_usd": 0.22216271, "judge_gross_standard_cost_usd": 0.0398325, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 328396, "agent_prompt_tokens": 316821, "agent_cached_input_tokens": 64183, "agent_thinking_tokens": 7300, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 190881, "cached_input_tokens": 10137, "completion_tokens": 3488, "thoughts_tokens": 3537, "total_tokens": 197906, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 125940, "cached_input_tokens": 54046, "completion_tokens": 787, "thoughts_tokens": 3763, "total_tokens": 130490, "request_count": 7 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 4, "main_agent_timed_model_calls": 6 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.525, 34.24], "agent_gross_standard_cost_usd": 0.6961994499999999, "judge_gross_standard_cost_usd": 0.061596, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 370792, "agent_prompt_tokens": 361241, "agent_cached_input_tokens": 20242, "agent_thinking_tokens": 5770, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 191451, "cached_input_tokens": 0, "completion_tokens": 2778, "thoughts_tokens": 5463, "total_tokens": 199692, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 167507, "cached_input_tokens": 20242, "completion_tokens": 915, "thoughts_tokens": 217, "total_tokens": 168639, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 33, "thoughts_tokens": 90, "total_tokens": 793, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1613, "cached_input_tokens": 0, "completion_tokens": 55, "thoughts_tokens": 0, "total_tokens": 1668, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 6, "main_agent_timed_model_calls": 6 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.392, 39.949], "agent_gross_standard_cost_usd": 0.29292015, "judge_gross_standard_cost_usd": 0.059553, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 353537, "agent_prompt_tokens": 339122, "agent_cached_input_tokens": 32797, "agent_thinking_tokens": 10795, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 191097, "cached_input_tokens": 0, "completion_tokens": 2829, "thoughts_tokens": 6746, "total_tokens": 200672, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 146377, "cached_input_tokens": 32797, "completion_tokens": 733, "thoughts_tokens": 4049, "total_tokens": 151159, "request_count": 7 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1648, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 1706, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 5, "main_agent_timed_model_calls": 6 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [105.791, 96.961, 18.114], "agent_gross_standard_cost_usd": 0.9651298150000001, "judge_gross_standard_cost_usd": 0.1179, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 659594, "agent_prompt_tokens": 646527, "agent_cached_input_tokens": 233118, "agent_thinking_tokens": 7412, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 205141, "cached_input_tokens": 10138, "completion_tokens": 3645, "thoughts_tokens": 6310, "total_tokens": 215096, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 405860, "cached_input_tokens": 208663, "completion_tokens": 1568, "thoughts_tokens": 0, "total_tokens": 407428, "request_count": 13 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32468, "cached_input_tokens": 14317, "completion_tokens": 251, "thoughts_tokens": 0, "total_tokens": 32719, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 3058, "cached_input_tokens": 0, "completion_tokens": 191, "thoughts_tokens": 1102, "total_tokens": 4351, "request_count": 4 } }, "agent_requests": 27, "main_agent_model_calls": 13, "tool_calls": 24, "main_agent_timed_model_calls": 13 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [114.342, 97.712, 21.744], "agent_gross_standard_cost_usd": 0.312738175, "judge_gross_standard_cost_usd": 0.093672, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 474504, "agent_prompt_tokens": 456692, "agent_cached_input_tokens": 141082, "agent_thinking_tokens": 12120, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 193270, "cached_input_tokens": 0, "completion_tokens": 3862, "thoughts_tokens": 5998, "total_tokens": 203130, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 230918, "cached_input_tokens": 133915, "completion_tokens": 1557, "thoughts_tokens": 6122, "total_tokens": 238597, "request_count": 13 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32504, "cached_input_tokens": 7167, "completion_tokens": 273, "thoughts_tokens": 0, "total_tokens": 32777, "request_count": 4 } }, "agent_requests": 23, "main_agent_model_calls": 9, "tool_calls": 14, "main_agent_timed_model_calls": 9 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [22.606], "agent_gross_standard_cost_usd": 0.1392962, "judge_gross_standard_cost_usd": 0.0286035, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109099, "agent_prompt_tokens": 105302, "agent_cached_input_tokens": 20116, "agent_thinking_tokens": 2438, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59265, "cached_input_tokens": 0, "completion_tokens": 910, "thoughts_tokens": 2438, "total_tokens": 62613, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 45379, "cached_input_tokens": 20116, "completion_tokens": 391, "thoughts_tokens": 0, "total_tokens": 45770, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [40.049], "agent_gross_standard_cost_usd": 0.12886055, "judge_gross_standard_cost_usd": 0.0293565, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 152188, "agent_prompt_tokens": 145027, "agent_cached_input_tokens": 38925, "agent_thinking_tokens": 5405, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59511, "cached_input_tokens": 0, "completion_tokens": 1072, "thoughts_tokens": 2385, "total_tokens": 62968, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 84858, "cached_input_tokens": 38925, "completion_tokens": 631, "thoughts_tokens": 3020, "total_tokens": 88509, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 53, "thoughts_tokens": 0, "total_tokens": 711, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [14.742], "agent_gross_standard_cost_usd": 0.10149720000000001, "judge_gross_standard_cost_usd": 0.0100575, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 69862, "agent_prompt_tokens": 66820, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2083, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48569, "cached_input_tokens": 0, "completion_tokens": 800, "thoughts_tokens": 1881, "total_tokens": 51250, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 18251, "cached_input_tokens": 0, "completion_tokens": 159, "thoughts_tokens": 202, "total_tokens": 18612, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [16.994], "agent_gross_standard_cost_usd": 0.0500505, "judge_gross_standard_cost_usd": 0.0099, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 66725, "agent_prompt_tokens": 63723, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2041, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48570, "cached_input_tokens": 0, "completion_tokens": 803, "thoughts_tokens": 1150, "total_tokens": 50523, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 15153, "cached_input_tokens": 0, "completion_tokens": 158, "thoughts_tokens": 891, "total_tokens": 16202, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.483], "agent_gross_standard_cost_usd": 0.17607791, "judge_gross_standard_cost_usd": 0.0243165, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 187946, "agent_prompt_tokens": 184822, "agent_cached_input_tokens": 82831, "agent_thinking_tokens": 2076, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 83940, "cached_input_tokens": 14207, "completion_tokens": 810, "thoughts_tokens": 2076, "total_tokens": 86826, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 100882, "cached_input_tokens": 68624, "completion_tokens": 238, "thoughts_tokens": 0, "total_tokens": 101120, "request_count": 5 } }, "agent_requests": 7, "main_agent_model_calls": 5, "tool_calls": 5, "main_agent_timed_model_calls": 5 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.184], "agent_gross_standard_cost_usd": 0.07543994, "judge_gross_standard_cost_usd": 0.0127815, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 113058, "agent_prompt_tokens": 108482, "agent_cached_input_tokens": 21030, "agent_thinking_tokens": 3567, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77102, "cached_input_tokens": 10148, "completion_tokens": 861, "thoughts_tokens": 1469, "total_tokens": 79432, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 31380, "cached_input_tokens": 10882, "completion_tokens": 148, "thoughts_tokens": 2098, "total_tokens": 33626, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 1, "main_agent_timed_model_calls": 2 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [49.062, 13.29], "agent_gross_standard_cost_usd": 0.49044525, "judge_gross_standard_cost_usd": 0.1928685, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 500677, "agent_prompt_tokens": 493347, "agent_cached_input_tokens": 181236, "agent_thinking_tokens": 3653, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 160906, "cached_input_tokens": 16340, "completion_tokens": 1732, "thoughts_tokens": 3653, "total_tokens": 166291, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 234791, "cached_input_tokens": 142082, "completion_tokens": 631, "thoughts_tokens": 0, "total_tokens": 235422, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 97650, "cached_input_tokens": 22814, "completion_tokens": 1314, "thoughts_tokens": 0, "total_tokens": 98964, "request_count": 10 } }, "agent_requests": 20, "main_agent_model_calls": 6, "tool_calls": 16, "main_agent_timed_model_calls": 6 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [72.551, 14.221], "agent_gross_standard_cost_usd": 0.238651, "judge_gross_standard_cost_usd": 0.1365855, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 398564, "agent_prompt_tokens": 386513, "agent_cached_input_tokens": 127766, "agent_thinking_tokens": 8864, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 145627, "cached_input_tokens": 0, "completion_tokens": 1700, "thoughts_tokens": 5086, "total_tokens": 152413, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 192432, "cached_input_tokens": 113646, "completion_tokens": 545, "thoughts_tokens": 3778, "total_tokens": 196755, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 48454, "cached_input_tokens": 14120, "completion_tokens": 942, "thoughts_tokens": 0, "total_tokens": 49396, "request_count": 6 } }, "agent_requests": 16, "main_agent_model_calls": 6, "tool_calls": 10, "main_agent_timed_model_calls": 6 }, { "task": "task-30", "suite": "Support hours", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [12.077], "agent_gross_standard_cost_usd": 0.020964399999999998, "judge_gross_standard_cost_usd": 0.012204, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48172, "agent_prompt_tokens": 44986, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 1916, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39138, "cached_input_tokens": 0, "completion_tokens": 1066, "thoughts_tokens": 1916, "total_tokens": 42120, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 204, "thoughts_tokens": 0, "total_tokens": 6052, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [13.683], "agent_gross_standard_cost_usd": 0.0227528, "judge_gross_standard_cost_usd": 0.01188, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48884, "agent_prompt_tokens": 44979, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2653, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39131, "cached_input_tokens": 0, "completion_tokens": 1041, "thoughts_tokens": 2653, "total_tokens": 42825, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 211, "thoughts_tokens": 0, "total_tokens": 6059, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "argon_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.404], "agent_gross_standard_cost_usd": 0.19036895, "judge_gross_standard_cost_usd": 0.074181, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 182019, "agent_prompt_tokens": 177234, "agent_cached_input_tokens": 64308, "agent_thinking_tokens": 2669, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67099, "cached_input_tokens": 14205, "completion_tokens": 1319, "thoughts_tokens": 2669, "total_tokens": 71087, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25178, "cached_input_tokens": 0, "completion_tokens": 409, "thoughts_tokens": 0, "total_tokens": 25587, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 84957, "cached_input_tokens": 50103, "completion_tokens": 388, "thoughts_tokens": 0, "total_tokens": 85345, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "baseline_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.812], "agent_gross_standard_cost_usd": 0.106449275, "judge_gross_standard_cost_usd": 0.077217, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 173828, "agent_prompt_tokens": 168639, "agent_cached_input_tokens": 46839, "agent_thinking_tokens": 3419, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68236, "cached_input_tokens": 0, "completion_tokens": 1002, "thoughts_tokens": 1809, "total_tokens": 71047, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25184, "cached_input_tokens": 7081, "completion_tokens": 457, "thoughts_tokens": 0, "total_tokens": 25641, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 75219, "cached_input_tokens": 39758, "completion_tokens": 311, "thoughts_tokens": 1610, "total_tokens": 77140, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.4], "agent_gross_standard_cost_usd": 0.1041386, "judge_gross_standard_cost_usd": 0.0784515, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 148244, "agent_prompt_tokens": 142059, "agent_cached_input_tokens": 21461, "agent_thinking_tokens": 4328, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68264, "cached_input_tokens": 0, "completion_tokens": 1092, "thoughts_tokens": 2608, "total_tokens": 71964, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25198, "cached_input_tokens": 7082, "completion_tokens": 403, "thoughts_tokens": 0, "total_tokens": 25601, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 48597, "cached_input_tokens": 14379, "completion_tokens": 362, "thoughts_tokens": 1720, "total_tokens": 50679, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.326], "agent_gross_standard_cost_usd": 0.133017675, "judge_gross_standard_cost_usd": 0.0763545, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 169934, "agent_prompt_tokens": 164258, "agent_cached_input_tokens": 37679, "agent_thinking_tokens": 3917, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66721, "cached_input_tokens": 14205, "completion_tokens": 996, "thoughts_tokens": 2430, "total_tokens": 70147, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25176, "cached_input_tokens": 7081, "completion_tokens": 412, "thoughts_tokens": 0, "total_tokens": 25588, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 72361, "cached_input_tokens": 16393, "completion_tokens": 351, "thoughts_tokens": 1487, "total_tokens": 74199, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.782], "agent_gross_standard_cost_usd": 0.204794425, "judge_gross_standard_cost_usd": 0.139212, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 313798, "agent_prompt_tokens": 304577, "agent_cached_input_tokens": 87352, "agent_thinking_tokens": 6584, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 76775, "cached_input_tokens": 0, "completion_tokens": 1357, "thoughts_tokens": 3558, "total_tokens": 81690, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 151947, "cached_input_tokens": 80193, "completion_tokens": 441, "thoughts_tokens": 3026, "total_tokens": 155414, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 75855, "cached_input_tokens": 7159, "completion_tokens": 839, "thoughts_tokens": 0, "total_tokens": 76694, "request_count": 9 } }, "agent_requests": 16, "main_agent_model_calls": 5, "tool_calls": 15, "main_agent_timed_model_calls": 5 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [75.111], "agent_gross_standard_cost_usd": 0.13769205, "judge_gross_standard_cost_usd": 0.112968, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 285381, "agent_prompt_tokens": 278725, "agent_cached_input_tokens": 101226, "agent_thinking_tokens": 4003, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 72263, "cached_input_tokens": 0, "completion_tokens": 1191, "thoughts_tokens": 2383, "total_tokens": 75837, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 96038, "cached_input_tokens": 33190, "completion_tokens": 1061, "thoughts_tokens": 0, "total_tokens": 97099, "request_count": 12 }, "gemini-3.7-flash": { "prompt_tokens": 110424, "cached_input_tokens": 68036, "completion_tokens": 401, "thoughts_tokens": 1620, "total_tokens": 112445, "request_count": 4 } }, "agent_requests": 18, "main_agent_model_calls": 4, "tool_calls": 16, "main_agent_timed_model_calls": 4 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [62.444], "agent_gross_standard_cost_usd": 0.180494025, "judge_gross_standard_cost_usd": 0.074664, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 241315, "agent_prompt_tokens": 232290, "agent_cached_input_tokens": 69645, "agent_thinking_tokens": 7132, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67083, "cached_input_tokens": 0, "completion_tokens": 1049, "thoughts_tokens": 2858, "total_tokens": 70990, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 129646, "cached_input_tokens": 64372, "completion_tokens": 335, "thoughts_tokens": 4274, "total_tokens": 134255, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 35561, "cached_input_tokens": 5273, "completion_tokens": 509, "thoughts_tokens": 0, "total_tokens": 36070, "request_count": 5 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 12, "main_agent_timed_model_calls": 5 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [43.264], "agent_gross_standard_cost_usd": 0.11548547499999999, "judge_gross_standard_cost_usd": 0.0577215, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 166492, "agent_prompt_tokens": 158482, "agent_cached_input_tokens": 51401, "agent_thinking_tokens": 5788, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60989, "cached_input_tokens": 0, "completion_tokens": 1519, "thoughts_tokens": 2471, "total_tokens": 64979, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 68056, "cached_input_tokens": 39146, "completion_tokens": 329, "thoughts_tokens": 3317, "total_tokens": 71702, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29437, "cached_input_tokens": 12255, "completion_tokens": 374, "thoughts_tokens": 0, "total_tokens": 29811, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [70.23], "agent_gross_standard_cost_usd": 0.20617315, "judge_gross_standard_cost_usd": 0.058794, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 297320, "agent_prompt_tokens": 290802, "agent_cached_input_tokens": 122791, "agent_thinking_tokens": 4602, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63320, "cached_input_tokens": 0, "completion_tokens": 1181, "thoughts_tokens": 1739, "total_tokens": 66240, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 206920, "cached_input_tokens": 122791, "completion_tokens": 434, "thoughts_tokens": 2863, "total_tokens": 210217, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20562, "cached_input_tokens": 0, "completion_tokens": 301, "thoughts_tokens": 0, "total_tokens": 20863, "request_count": 3 } }, "agent_requests": 13, "main_agent_model_calls": 8, "tool_calls": 11, "main_agent_timed_model_calls": 8 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.933], "agent_gross_standard_cost_usd": 0.101575525, "judge_gross_standard_cost_usd": 0.0565275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 164759, "agent_prompt_tokens": 158262, "agent_cached_input_tokens": 51271, "agent_thinking_tokens": 5083, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60845, "cached_input_tokens": 0, "completion_tokens": 794, "thoughts_tokens": 2641, "total_tokens": 64280, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 68076, "cached_input_tokens": 39022, "completion_tokens": 246, "thoughts_tokens": 2442, "total_tokens": 70764, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29341, "cached_input_tokens": 12249, "completion_tokens": 374, "thoughts_tokens": 0, "total_tokens": 29715, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [38.091, 32.864], "agent_gross_standard_cost_usd": 0.19944635, "judge_gross_standard_cost_usd": 0.107229, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 367328, "agent_prompt_tokens": 357100, "agent_cached_input_tokens": 109439, "agent_thinking_tokens": 6680, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 137625, "cached_input_tokens": 0, "completion_tokens": 2190, "thoughts_tokens": 4644, "total_tokens": 144459, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 72716, "cached_input_tokens": 27040, "completion_tokens": 896, "thoughts_tokens": 0, "total_tokens": 73612, "request_count": 10 }, "gemini-3.7-flash": { "prompt_tokens": 146759, "cached_input_tokens": 82399, "completion_tokens": 462, "thoughts_tokens": 2036, "total_tokens": 149257, "request_count": 6 } }, "agent_requests": 20, "main_agent_model_calls": 6, "tool_calls": 13, "main_agent_timed_model_calls": 6 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [62.866, 88.287], "agent_gross_standard_cost_usd": 0.347129625, "judge_gross_standard_cost_usd": 0.109629, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 524387, "agent_prompt_tokens": 504030, "agent_cached_input_tokens": 156638, "agent_thinking_tokens": 16535, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180146, "cached_input_tokens": 0, "completion_tokens": 2483, "thoughts_tokens": 8943, "total_tokens": 191572, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 88675, "cached_input_tokens": 22143, "completion_tokens": 572, "thoughts_tokens": 0, "total_tokens": 89247, "request_count": 11 }, "gemini-3.5-flash": { "prompt_tokens": 235209, "cached_input_tokens": 134495, "completion_tokens": 767, "thoughts_tokens": 7592, "total_tokens": 243568, "request_count": 11 } }, "agent_requests": 27, "main_agent_model_calls": 11, "tool_calls": 24, "main_agent_timed_model_calls": 11 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.343], "agent_gross_standard_cost_usd": 0.00480708, "judge_gross_standard_cost_usd": 0.0093525, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16619, "agent_prompt_tokens": 15455, "agent_cached_input_tokens": 10146, "agent_thinking_tokens": 686, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15455, "cached_input_tokens": 10146, "completion_tokens": 478, "thoughts_tokens": 686, "total_tokens": 16619, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.079], "agent_gross_standard_cost_usd": 0.007409300000000001, "judge_gross_standard_cost_usd": 0.009507, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16565, "agent_prompt_tokens": 15456, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 636, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15456, "cached_input_tokens": 0, "completion_tokens": 473, "thoughts_tokens": 636, "total_tokens": 16565, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.944], "agent_gross_standard_cost_usd": 0.08670375000000001, "judge_gross_standard_cost_usd": 0.0271185, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 125693, "agent_prompt_tokens": 120873, "agent_cached_input_tokens": 36257, "agent_thinking_tokens": 3881, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55449, "cached_input_tokens": 0, "completion_tokens": 722, "thoughts_tokens": 2332, "total_tokens": 58503, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 65424, "cached_input_tokens": 36257, "completion_tokens": 217, "thoughts_tokens": 1549, "total_tokens": 67190, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [20.734], "agent_gross_standard_cost_usd": 0.08865005, "judge_gross_standard_cost_usd": 0.031725, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 135049, "agent_prompt_tokens": 131381, "agent_cached_input_tokens": 42439, "agent_thinking_tokens": 2741, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58829, "cached_input_tokens": 0, "completion_tokens": 707, "thoughts_tokens": 1377, "total_tokens": 60913, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 72552, "cached_input_tokens": 42439, "completion_tokens": 220, "thoughts_tokens": 1364, "total_tokens": 74136, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [40.485], "agent_gross_standard_cost_usd": 0.07750654, "judge_gross_standard_cost_usd": 0.021249, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 103175, "agent_prompt_tokens": 97060, "agent_cached_input_tokens": 45058, "agent_thinking_tokens": 4980, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48531, "cached_input_tokens": 16313, "completion_tokens": 901, "thoughts_tokens": 2433, "total_tokens": 51865, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 48529, "cached_input_tokens": 28745, "completion_tokens": 234, "thoughts_tokens": 2547, "total_tokens": 51310, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.207], "agent_gross_standard_cost_usd": 0.1079631, "judge_gross_standard_cost_usd": 0.026025, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 127718, "agent_prompt_tokens": 123012, "agent_cached_input_tokens": 25026, "agent_thinking_tokens": 3766, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52779, "cached_input_tokens": 0, "completion_tokens": 719, "thoughts_tokens": 2227, "total_tokens": 55725, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 70233, "cached_input_tokens": 25026, "completion_tokens": 221, "thoughts_tokens": 1539, "total_tokens": 71993, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.506], "agent_gross_standard_cost_usd": 0.07997145, "judge_gross_standard_cost_usd": 0.024516, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 117374, "agent_prompt_tokens": 113068, "agent_cached_input_tokens": 36069, "agent_thinking_tokens": 3236, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51002, "cached_input_tokens": 0, "completion_tokens": 856, "thoughts_tokens": 1550, "total_tokens": 53408, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 62066, "cached_input_tokens": 36069, "completion_tokens": 214, "thoughts_tokens": 1686, "total_tokens": 63966, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.356], "agent_gross_standard_cost_usd": 0.09945124999999999, "judge_gross_standard_cost_usd": 0.0329895, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 150157, "agent_prompt_tokens": 144450, "agent_cached_input_tokens": 58213, "agent_thinking_tokens": 4418, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55261, "cached_input_tokens": 0, "completion_tokens": 1046, "thoughts_tokens": 2598, "total_tokens": 58905, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 89189, "cached_input_tokens": 58213, "completion_tokens": 243, "thoughts_tokens": 1820, "total_tokens": 91252, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [51.951], "agent_gross_standard_cost_usd": 0.09341675, "judge_gross_standard_cost_usd": 0.023049, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 151975, "agent_prompt_tokens": 146506, "agent_cached_input_tokens": 46079, "agent_thinking_tokens": 4054, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77148, "cached_input_tokens": 0, "completion_tokens": 1100, "thoughts_tokens": 2040, "total_tokens": 80288, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 68676, "cached_input_tokens": 46079, "completion_tokens": 260, "thoughts_tokens": 1823, "total_tokens": 70759, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 55, "thoughts_tokens": 191, "total_tokens": 928, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.479], "agent_gross_standard_cost_usd": 0.0846343, "judge_gross_standard_cost_usd": 0.021951, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116193, "agent_prompt_tokens": 110022, "agent_cached_input_tokens": 12820, "agent_thinking_tokens": 4830, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77141, "cached_input_tokens": 0, "completion_tokens": 1026, "thoughts_tokens": 2335, "total_tokens": 80502, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 32199, "cached_input_tokens": 12820, "completion_tokens": 261, "thoughts_tokens": 2188, "total_tokens": 34648, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 54, "thoughts_tokens": 307, "total_tokens": 1043, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [215.128], "agent_gross_standard_cost_usd": 0.39975575, "judge_gross_standard_cost_usd": 0.424011, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 629643, "agent_prompt_tokens": 619695, "agent_cached_input_tokens": 130259, "agent_thinking_tokens": 6035, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 144258, "cached_input_tokens": 0, "completion_tokens": 1015, "thoughts_tokens": 2662, "total_tokens": 147935, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 226998, "cached_input_tokens": 52353, "completion_tokens": 540, "thoughts_tokens": 3373, "total_tokens": 230911, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 248439, "cached_input_tokens": 77906, "completion_tokens": 2358, "thoughts_tokens": 0, "total_tokens": 250797, "request_count": 18 } }, "agent_requests": 26, "main_agent_model_calls": 6, "tool_calls": 26, "main_agent_timed_model_calls": 6 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [124.84], "agent_gross_standard_cost_usd": 0.3368376, "judge_gross_standard_cost_usd": 0.4666635, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 564240, "agent_prompt_tokens": 552355, "agent_cached_input_tokens": 117492, "agent_thinking_tokens": 8097, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 149931, "cached_input_tokens": 0, "completion_tokens": 1125, "thoughts_tokens": 3777, "total_tokens": 154833, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 179188, "cached_input_tokens": 54888, "completion_tokens": 503, "thoughts_tokens": 3860, "total_tokens": 183551, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 223236, "cached_input_tokens": 62604, "completion_tokens": 2160, "thoughts_tokens": 460, "total_tokens": 225856, "request_count": 15 } }, "agent_requests": 22, "main_agent_model_calls": 5, "tool_calls": 17, "main_agent_timed_model_calls": 5 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [126.791], "agent_gross_standard_cost_usd": 0.44364565, "judge_gross_standard_cost_usd": 0.2697795, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 681075, "agent_prompt_tokens": 663194, "agent_cached_input_tokens": 86475, "agent_thinking_tokens": 11682, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 229403, "cached_input_tokens": 0, "completion_tokens": 1727, "thoughts_tokens": 5201, "total_tokens": 236331, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 217761, "cached_input_tokens": 22270, "completion_tokens": 3716, "thoughts_tokens": 0, "total_tokens": 221477, "request_count": 18 }, "gemini-3.5-flash": { "prompt_tokens": 216030, "cached_input_tokens": 64205, "completion_tokens": 756, "thoughts_tokens": 6481, "total_tokens": 223267, "request_count": 6 } }, "agent_requests": 27, "main_agent_model_calls": 6, "tool_calls": 16, "main_agent_timed_model_calls": 6 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [100.966], "agent_gross_standard_cost_usd": 0.316181375, "judge_gross_standard_cost_usd": 0.1834425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 623388, "agent_prompt_tokens": 608484, "agent_cached_input_tokens": 163155, "agent_thinking_tokens": 9308, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 201440, "cached_input_tokens": 0, "completion_tokens": 1703, "thoughts_tokens": 5484, "total_tokens": 208627, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 229116, "cached_input_tokens": 84957, "completion_tokens": 3264, "thoughts_tokens": 0, "total_tokens": 232380, "request_count": 17 }, "gemini-3.7-flash": { "prompt_tokens": 177928, "cached_input_tokens": 78198, "completion_tokens": 629, "thoughts_tokens": 3824, "total_tokens": 182381, "request_count": 5 } }, "agent_requests": 25, "main_agent_model_calls": 5, "tool_calls": 17, "main_agent_timed_model_calls": 5 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [94.587], "agent_gross_standard_cost_usd": 0.205767125, "judge_gross_standard_cost_usd": 0.2831085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 428199, "agent_prompt_tokens": 418815, "agent_cached_input_tokens": 130214, "agent_thinking_tokens": 6362, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 118738, "cached_input_tokens": 0, "completion_tokens": 1074, "thoughts_tokens": 2534, "total_tokens": 122346, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 148529, "cached_input_tokens": 93341, "completion_tokens": 520, "thoughts_tokens": 3828, "total_tokens": 152877, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 151548, "cached_input_tokens": 36873, "completion_tokens": 1428, "thoughts_tokens": 0, "total_tokens": 152976, "request_count": 12 } }, "agent_requests": 19, "main_agent_model_calls": 5, "tool_calls": 18, "main_agent_timed_model_calls": 5 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [130.04], "agent_gross_standard_cost_usd": 0.33071185, "judge_gross_standard_cost_usd": 0.508155, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 657884, "agent_prompt_tokens": 645382, "agent_cached_input_tokens": 210179, "agent_thinking_tokens": 8605, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 156215, "cached_input_tokens": 0, "completion_tokens": 1053, "thoughts_tokens": 4053, "total_tokens": 161321, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 244174, "cached_input_tokens": 140265, "completion_tokens": 457, "thoughts_tokens": 4552, "total_tokens": 249183, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 244993, "cached_input_tokens": 69914, "completion_tokens": 2387, "thoughts_tokens": 0, "total_tokens": 247380, "request_count": 17 } }, "agent_requests": 25, "main_agent_model_calls": 6, "tool_calls": 22, "main_agent_timed_model_calls": 6 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [66.775], "agent_gross_standard_cost_usd": 0.12151767499999999, "judge_gross_standard_cost_usd": 0.039459, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 264429, "agent_prompt_tokens": 254482, "agent_cached_input_tokens": 56529, "agent_thinking_tokens": 5003, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54559, "cached_input_tokens": 0, "completion_tokens": 1001, "thoughts_tokens": 3016, "total_tokens": 58576, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 55594, "cached_input_tokens": 33226, "completion_tokens": 258, "thoughts_tokens": 1987, "total_tokens": 57839, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 144329, "cached_input_tokens": 23303, "completion_tokens": 3685, "thoughts_tokens": 0, "total_tokens": 148014, "request_count": 18 } }, "agent_requests": 23, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.45], "agent_gross_standard_cost_usd": 0.0829243, "judge_gross_standard_cost_usd": 0.026067, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 101749, "agent_prompt_tokens": 96292, "agent_cached_input_tokens": 14476, "agent_thinking_tokens": 4472, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56353, "cached_input_tokens": 0, "completion_tokens": 805, "thoughts_tokens": 2250, "total_tokens": 59408, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 39939, "cached_input_tokens": 14476, "completion_tokens": 180, "thoughts_tokens": 2222, "total_tokens": 42341, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.986], "agent_gross_standard_cost_usd": 0.0857639, "judge_gross_standard_cost_usd": 0.0373185, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 139854, "agent_prompt_tokens": 135473, "agent_cached_input_tokens": 41137, "agent_thinking_tokens": 2886, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57777, "cached_input_tokens": 0, "completion_tokens": 1035, "thoughts_tokens": 1753, "total_tokens": 60565, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12913, "cached_input_tokens": 5116, "completion_tokens": 185, "thoughts_tokens": 0, "total_tokens": 13098, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 64783, "cached_input_tokens": 36021, "completion_tokens": 275, "thoughts_tokens": 1133, "total_tokens": 66191, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.105], "agent_gross_standard_cost_usd": 0.07477255000000001, "judge_gross_standard_cost_usd": 0.0309585, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 133192, "agent_prompt_tokens": 128105, "agent_cached_input_tokens": 49163, "agent_thinking_tokens": 3465, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51542, "cached_input_tokens": 0, "completion_tokens": 1117, "thoughts_tokens": 2138, "total_tokens": 54797, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 23878, "cached_input_tokens": 16000, "completion_tokens": 259, "thoughts_tokens": 0, "total_tokens": 24137, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 52685, "cached_input_tokens": 33163, "completion_tokens": 246, "thoughts_tokens": 1327, "total_tokens": 54258, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.928], "agent_gross_standard_cost_usd": 0.058446325, "judge_gross_standard_cost_usd": 0.028044, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 121798, "agent_prompt_tokens": 117083, "agent_cached_input_tokens": 50243, "agent_thinking_tokens": 3463, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51510, "cached_input_tokens": 0, "completion_tokens": 807, "thoughts_tokens": 1998, "total_tokens": 54315, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12904, "cached_input_tokens": 5115, "completion_tokens": 182, "thoughts_tokens": 0, "total_tokens": 13086, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 52669, "cached_input_tokens": 45128, "completion_tokens": 263, "thoughts_tokens": 1465, "total_tokens": 54397, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.951], "agent_gross_standard_cost_usd": 0.08187219, "judge_gross_standard_cost_usd": 0.0395325, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 141536, "agent_prompt_tokens": 137185, "agent_cached_input_tokens": 57679, "agent_thinking_tokens": 2985, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57767, "cached_input_tokens": 16328, "completion_tokens": 910, "thoughts_tokens": 1805, "total_tokens": 60482, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14514, "cached_input_tokens": 5316, "completion_tokens": 208, "thoughts_tokens": 0, "total_tokens": 14722, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 64904, "cached_input_tokens": 36035, "completion_tokens": 248, "thoughts_tokens": 1180, "total_tokens": 66332, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [69.307], "agent_gross_standard_cost_usd": 0.1434347, "judge_gross_standard_cost_usd": 0.0478575, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 214351, "agent_prompt_tokens": 205026, "agent_cached_input_tokens": 87806, "agent_thinking_tokens": 7708, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61826, "cached_input_tokens": 0, "completion_tokens": 980, "thoughts_tokens": 2825, "total_tokens": 65631, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16390, "cached_input_tokens": 0, "completion_tokens": 300, "thoughts_tokens": 0, "total_tokens": 16690, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 126810, "cached_input_tokens": 87806, "completion_tokens": 337, "thoughts_tokens": 4883, "total_tokens": 132030, "request_count": 5 } }, "agent_requests": 11, "main_agent_model_calls": 5, "tool_calls": 10, "main_agent_timed_model_calls": 5 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.477], "agent_gross_standard_cost_usd": 0.08310835, "judge_gross_standard_cost_usd": 0.0295275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 125374, "agent_prompt_tokens": 119514, "agent_cached_input_tokens": 36820, "agent_thinking_tokens": 4359, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50467, "cached_input_tokens": 0, "completion_tokens": 930, "thoughts_tokens": 2254, "total_tokens": 53651, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17052, "cached_input_tokens": 5354, "completion_tokens": 353, "thoughts_tokens": 0, "total_tokens": 17405, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 51995, "cached_input_tokens": 31466, "completion_tokens": 218, "thoughts_tokens": 2105, "total_tokens": 54318, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.367], "agent_gross_standard_cost_usd": 0.154785375, "judge_gross_standard_cost_usd": 0.056415, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 163900, "agent_prompt_tokens": 156744, "agent_cached_input_tokens": 23817, "agent_thinking_tokens": 5537, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63149, "cached_input_tokens": 0, "completion_tokens": 1049, "thoughts_tokens": 2290, "total_tokens": 66488, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 75844, "cached_input_tokens": 16582, "completion_tokens": 430, "thoughts_tokens": 3247, "total_tokens": 79521, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17751, "cached_input_tokens": 7235, "completion_tokens": 140, "thoughts_tokens": 0, "total_tokens": 17891, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.481], "agent_gross_standard_cost_usd": 0.08989415, "judge_gross_standard_cost_usd": 0.031233, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140900, "agent_prompt_tokens": 133819, "agent_cached_input_tokens": 54159, "agent_thinking_tokens": 5959, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59431, "cached_input_tokens": 0, "completion_tokens": 846, "thoughts_tokens": 3056, "total_tokens": 63333, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 74388, "cached_input_tokens": 54159, "completion_tokens": 276, "thoughts_tokens": 2903, "total_tokens": 77567, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [44.187], "agent_gross_standard_cost_usd": 0.1182151, "judge_gross_standard_cost_usd": 0.0378765, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 188896, "agent_prompt_tokens": 182350, "agent_cached_input_tokens": 82340, "agent_thinking_tokens": 4640, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 64142, "cached_input_tokens": 0, "completion_tokens": 1138, "thoughts_tokens": 1883, "total_tokens": 67163, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 117550, "cached_input_tokens": 82340, "completion_tokens": 710, "thoughts_tokens": 2757, "total_tokens": 121017, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 6, "main_agent_timed_model_calls": 5 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [69.304], "agent_gross_standard_cost_usd": 0.11597640000000001, "judge_gross_standard_cost_usd": 0.032211, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 175187, "agent_prompt_tokens": 167838, "agent_cached_input_tokens": 79874, "agent_thinking_tokens": 5779, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59541, "cached_input_tokens": 0, "completion_tokens": 974, "thoughts_tokens": 2316, "total_tokens": 62831, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 107639, "cached_input_tokens": 79874, "completion_tokens": 538, "thoughts_tokens": 3463, "total_tokens": 111640, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 5, "main_agent_timed_model_calls": 5 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [46.513], "agent_gross_standard_cost_usd": 0.15933575, "judge_gross_standard_cost_usd": 0.052143, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 169013, "agent_prompt_tokens": 160371, "agent_cached_input_tokens": 37775, "agent_thinking_tokens": 6946, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 65255, "cached_input_tokens": 0, "completion_tokens": 1184, "thoughts_tokens": 3017, "total_tokens": 69456, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1186, "cached_input_tokens": 0, "completion_tokens": 121, "thoughts_tokens": 0, "total_tokens": 1307, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 93930, "cached_input_tokens": 37775, "completion_tokens": 391, "thoughts_tokens": 3929, "total_tokens": 98250, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [32.726], "agent_gross_standard_cost_usd": 0.08463654999999999, "judge_gross_standard_cost_usd": 0.0291555, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 124026, "agent_prompt_tokens": 118431, "agent_cached_input_tokens": 35869, "agent_thinking_tokens": 4013, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56309, "cached_input_tokens": 0, "completion_tokens": 1190, "thoughts_tokens": 2375, "total_tokens": 59874, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 61468, "cached_input_tokens": 35869, "completion_tokens": 336, "thoughts_tokens": 1638, "total_tokens": 63442, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [44.882], "agent_gross_standard_cost_usd": 0.11466609999999999, "judge_gross_standard_cost_usd": 0.0229335, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 153266, "agent_prompt_tokens": 144958, "agent_cached_input_tokens": 53052, "agent_thinking_tokens": 6615, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59371, "cached_input_tokens": 0, "completion_tokens": 1056, "thoughts_tokens": 3153, "total_tokens": 63580, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 84929, "cached_input_tokens": 53052, "completion_tokens": 579, "thoughts_tokens": 3462, "total_tokens": 88970, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.411], "agent_gross_standard_cost_usd": 0.08244085, "judge_gross_standard_cost_usd": 0.022395, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 104554, "agent_prompt_tokens": 98407, "agent_cached_input_tokens": 15919, "agent_thinking_tokens": 4789, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59110, "cached_input_tokens": 0, "completion_tokens": 967, "thoughts_tokens": 3158, "total_tokens": 63235, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 38639, "cached_input_tokens": 15919, "completion_tokens": 333, "thoughts_tokens": 1631, "total_tokens": 40603, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [21.848], "agent_gross_standard_cost_usd": 0.0512696, "judge_gross_standard_cost_usd": 0.010269, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 67065, "agent_prompt_tokens": 63729, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2405, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48572, "cached_input_tokens": 0, "completion_tokens": 767, "thoughts_tokens": 1704, "total_tokens": 51043, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 15157, "cached_input_tokens": 0, "completion_tokens": 164, "thoughts_tokens": 701, "total_tokens": 16022, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [17.229], "agent_gross_standard_cost_usd": 0.052001399999999996, "judge_gross_standard_cost_usd": 0.010713, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 67336, "agent_prompt_tokens": 63709, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2664, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48558, "cached_input_tokens": 0, "completion_tokens": 808, "thoughts_tokens": 1691, "total_tokens": 51057, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 15151, "cached_input_tokens": 0, "completion_tokens": 155, "thoughts_tokens": 973, "total_tokens": 16279, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.4, 17.166], "agent_gross_standard_cost_usd": 0.2233518, "judge_gross_standard_cost_usd": 0.07464, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 321108, "agent_prompt_tokens": 309949, "agent_cached_input_tokens": 30331, "agent_thinking_tokens": 7816, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 122198, "cached_input_tokens": 0, "completion_tokens": 1701, "thoughts_tokens": 3788, "total_tokens": 127687, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 92434, "cached_input_tokens": 14527, "completion_tokens": 477, "thoughts_tokens": 3715, "total_tokens": 96626, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 95317, "cached_input_tokens": 15804, "completion_tokens": 1165, "thoughts_tokens": 313, "total_tokens": 96795, "request_count": 10 } }, "agent_requests": 18, "main_agent_model_calls": 4, "tool_calls": 11, "main_agent_timed_model_calls": 4 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [50.045, 14.071], "agent_gross_standard_cost_usd": 0.236388525, "judge_gross_standard_cost_usd": 0.109176, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 277169, "agent_prompt_tokens": 264644, "agent_cached_input_tokens": 25176, "agent_thinking_tokens": 9636, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 134535, "cached_input_tokens": 0, "completion_tokens": 1867, "thoughts_tokens": 4921, "total_tokens": 141323, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 99835, "cached_input_tokens": 18199, "completion_tokens": 521, "thoughts_tokens": 4715, "total_tokens": 105071, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 30274, "cached_input_tokens": 6977, "completion_tokens": 501, "thoughts_tokens": 0, "total_tokens": 30775, "request_count": 3 } }, "agent_requests": 11, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.113], "agent_gross_standard_cost_usd": 0.1041928, "judge_gross_standard_cost_usd": 0.026988, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 175703, "agent_prompt_tokens": 171224, "agent_cached_input_tokens": 58555, "agent_thinking_tokens": 3457, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 82986, "cached_input_tokens": 0, "completion_tokens": 758, "thoughts_tokens": 1214, "total_tokens": 84958, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 903, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 953, "request_count": 1 }, "gemini-3.5-flash": { "prompt_tokens": 87335, "cached_input_tokens": 58555, "completion_tokens": 214, "thoughts_tokens": 2243, "total_tokens": 89792, "request_count": 4 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.304], "agent_gross_standard_cost_usd": 0.09925271000000001, "judge_gross_standard_cost_usd": 0.02832, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 170924, "agent_prompt_tokens": 166495, "agent_cached_input_tokens": 66335, "agent_thinking_tokens": 3383, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 84303, "cached_input_tokens": 14207, "completion_tokens": 720, "thoughts_tokens": 947, "total_tokens": 85970, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 82192, "cached_input_tokens": 52128, "completion_tokens": 326, "thoughts_tokens": 2436, "total_tokens": 84954, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.142], "agent_gross_standard_cost_usd": 0.06823587, "judge_gross_standard_cost_usd": 0.0152175, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 92108, "agent_prompt_tokens": 87080, "agent_cached_input_tokens": 24175, "agent_thinking_tokens": 4076, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52299, "cached_input_tokens": 10149, "completion_tokens": 754, "thoughts_tokens": 2378, "total_tokens": 55431, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 34781, "cached_input_tokens": 14026, "completion_tokens": 198, "thoughts_tokens": 1698, "total_tokens": 36677, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [147.533], "agent_gross_standard_cost_usd": 0.17950195, "judge_gross_standard_cost_usd": 0.0305085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 286643, "agent_prompt_tokens": 276518, "agent_cached_input_tokens": 133575, "agent_thinking_tokens": 8335, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 92164, "cached_input_tokens": 0, "completion_tokens": 1179, "thoughts_tokens": 4279, "total_tokens": 97622, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 184354, "cached_input_tokens": 133575, "completion_tokens": 611, "thoughts_tokens": 4056, "total_tokens": 189021, "request_count": 10 } }, "agent_requests": 13, "main_agent_model_calls": 10, "tool_calls": 9, "main_agent_timed_model_calls": 10 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [58.33], "agent_gross_standard_cost_usd": 0.10241494999999999, "judge_gross_standard_cost_usd": 0.024762, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 126473, "agent_prompt_tokens": 119418, "agent_cached_input_tokens": 38061, "agent_thinking_tokens": 5633, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55371, "cached_input_tokens": 0, "completion_tokens": 1126, "thoughts_tokens": 2317, "total_tokens": 58814, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 64047, "cached_input_tokens": 38061, "completion_tokens": 296, "thoughts_tokens": 3316, "total_tokens": 67659, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.687], "agent_gross_standard_cost_usd": 0.09630425, "judge_gross_standard_cost_usd": 0.022449, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 138176, "agent_prompt_tokens": 131263, "agent_cached_input_tokens": 49605, "agent_thinking_tokens": 5754, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52830, "cached_input_tokens": 0, "completion_tokens": 891, "thoughts_tokens": 3524, "total_tokens": 57245, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 78433, "cached_input_tokens": 49605, "completion_tokens": 268, "thoughts_tokens": 2230, "total_tokens": 80931, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [73.013], "agent_gross_standard_cost_usd": 0.15778365, "judge_gross_standard_cost_usd": 0.051924, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 248917, "agent_prompt_tokens": 238301, "agent_cached_input_tokens": 114554, "agent_thinking_tokens": 8758, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63761, "cached_input_tokens": 0, "completion_tokens": 1154, "thoughts_tokens": 2800, "total_tokens": 67715, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 140625, "cached_input_tokens": 100082, "completion_tokens": 330, "thoughts_tokens": 5958, "total_tokens": 146913, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 33915, "cached_input_tokens": 14472, "completion_tokens": 374, "thoughts_tokens": 0, "total_tokens": 34289, "request_count": 4 } }, "agent_requests": 11, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [50.307], "agent_gross_standard_cost_usd": 0.11654345, "judge_gross_standard_cost_usd": 0.0294405, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 128480, "agent_prompt_tokens": 120279, "agent_cached_input_tokens": 30109, "agent_thinking_tokens": 6657, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58897, "cached_input_tokens": 0, "completion_tokens": 1207, "thoughts_tokens": 2782, "total_tokens": 62886, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 60710, "cached_input_tokens": 30109, "completion_tokens": 292, "thoughts_tokens": 3632, "total_tokens": 64634, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 45, "thoughts_tokens": 243, "total_tokens": 960, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 4.0, "customer_turn_seconds": [62.731], "agent_gross_standard_cost_usd": 0.1083761, "judge_gross_standard_cost_usd": 0.0337815, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 154752, "agent_prompt_tokens": 147530, "agent_cached_input_tokens": 60372, "agent_thinking_tokens": 5471, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59656, "cached_input_tokens": 0, "completion_tokens": 1250, "thoughts_tokens": 2155, "total_tokens": 63061, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 86378, "cached_input_tokens": 60372, "completion_tokens": 391, "thoughts_tokens": 3316, "total_tokens": 90085, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 8, "main_agent_timed_model_calls": 4 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.171], "agent_gross_standard_cost_usd": 0.10387805, "judge_gross_standard_cost_usd": 0.034584, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 143913, "agent_prompt_tokens": 136272, "agent_cached_input_tokens": 41467, "agent_thinking_tokens": 5884, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63835, "cached_input_tokens": 0, "completion_tokens": 1250, "thoughts_tokens": 3295, "total_tokens": 68380, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 70941, "cached_input_tokens": 41467, "completion_tokens": 397, "thoughts_tokens": 2589, "total_tokens": 73927, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.225, 24.872, 19.629], "agent_gross_standard_cost_usd": 0.20044805, "judge_gross_standard_cost_usd": 0.0213945, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 342339, "agent_prompt_tokens": 330712, "agent_cached_input_tokens": 92356, "agent_thinking_tokens": 7805, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 179613, "cached_input_tokens": 0, "completion_tokens": 2954, "thoughts_tokens": 4884, "total_tokens": 187451, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1989, "cached_input_tokens": 0, "completion_tokens": 155, "thoughts_tokens": 0, "total_tokens": 2144, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 149110, "cached_input_tokens": 92356, "completion_tokens": 713, "thoughts_tokens": 2921, "total_tokens": 152744, "request_count": 8 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 7, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [57.604, 28.348, 18.104], "agent_gross_standard_cost_usd": 0.21012385, "judge_gross_standard_cost_usd": 0.0207945, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 325463, "agent_prompt_tokens": 313948, "agent_cached_input_tokens": 68863, "agent_thinking_tokens": 7465, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180433, "cached_input_tokens": 0, "completion_tokens": 3173, "thoughts_tokens": 4691, "total_tokens": 188297, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1986, "cached_input_tokens": 0, "completion_tokens": 180, "thoughts_tokens": 0, "total_tokens": 2166, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 131529, "cached_input_tokens": 68863, "completion_tokens": 697, "thoughts_tokens": 2774, "total_tokens": 135000, "request_count": 7 } }, "agent_requests": 16, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [11.646], "agent_gross_standard_cost_usd": 0.0200093, "judge_gross_standard_cost_usd": 0.012153, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 47787, "agent_prompt_tokens": 44979, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 1573, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39131, "cached_input_tokens": 0, "completion_tokens": 1023, "thoughts_tokens": 1573, "total_tokens": 41727, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 212, "thoughts_tokens": 0, "total_tokens": 6060, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [13.479], "agent_gross_standard_cost_usd": 0.0216503, "judge_gross_standard_cost_usd": 0.011757, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48443, "agent_prompt_tokens": 44979, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2318, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39131, "cached_input_tokens": 0, "completion_tokens": 935, "thoughts_tokens": 2318, "total_tokens": 42384, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 211, "thoughts_tokens": 0, "total_tokens": 6059, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.206], "agent_gross_standard_cost_usd": 0.101250025, "judge_gross_standard_cost_usd": 0.06603, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 166670, "agent_prompt_tokens": 160741, "agent_cached_input_tokens": 55204, "agent_thinking_tokens": 4299, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61706, "cached_input_tokens": 0, "completion_tokens": 979, "thoughts_tokens": 2287, "total_tokens": 64972, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 26432, "cached_input_tokens": 13989, "completion_tokens": 354, "thoughts_tokens": 0, "total_tokens": 26786, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 72603, "cached_input_tokens": 41215, "completion_tokens": 297, "thoughts_tokens": 2012, "total_tokens": 74912, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.518], "agent_gross_standard_cost_usd": 0.09671402500000001, "judge_gross_standard_cost_usd": 0.0581745, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 151292, "agent_prompt_tokens": 146051, "agent_cached_input_tokens": 44861, "agent_thinking_tokens": 3647, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60920, "cached_input_tokens": 0, "completion_tokens": 1084, "thoughts_tokens": 1903, "total_tokens": 63907, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 18134, "cached_input_tokens": 7047, "completion_tokens": 228, "thoughts_tokens": 0, "total_tokens": 18362, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 66997, "cached_input_tokens": 37814, "completion_tokens": 282, "thoughts_tokens": 1744, "total_tokens": 69023, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [36.932, 24.578], "agent_gross_standard_cost_usd": 0.18365884999999998, "judge_gross_standard_cost_usd": 0.0619155, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 281519, "agent_prompt_tokens": 270224, "agent_cached_input_tokens": 91699, "agent_thinking_tokens": 8287, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 111485, "cached_input_tokens": 0, "completion_tokens": 2176, "thoughts_tokens": 4320, "total_tokens": 117981, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17722, "cached_input_tokens": 0, "completion_tokens": 184, "thoughts_tokens": 0, "total_tokens": 17906, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 141017, "cached_input_tokens": 91699, "completion_tokens": 648, "thoughts_tokens": 3967, "total_tokens": 145632, "request_count": 7 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 8, "main_agent_timed_model_calls": 7 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [32.334, 30.651], "agent_gross_standard_cost_usd": 0.21020914999999998, "judge_gross_standard_cost_usd": 0.107193, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 314281, "agent_prompt_tokens": 302736, "agent_cached_input_tokens": 88668, "agent_thinking_tokens": 8259, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 126964, "cached_input_tokens": 0, "completion_tokens": 2350, "thoughts_tokens": 4442, "total_tokens": 133756, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 22973, "cached_input_tokens": 5268, "completion_tokens": 279, "thoughts_tokens": 0, "total_tokens": 23252, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 152799, "cached_input_tokens": 83400, "completion_tokens": 657, "thoughts_tokens": 3817, "total_tokens": 157273, "request_count": 6 } }, "agent_requests": 14, "main_agent_model_calls": 6, "tool_calls": 15, "main_agent_timed_model_calls": 6 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [128.945, 96.78, 23.453], "agent_gross_standard_cost_usd": 0.34423695, "judge_gross_standard_cost_usd": 0.0924435, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 572368, "agent_prompt_tokens": 554978, "agent_cached_input_tokens": 221593, "agent_thinking_tokens": 11875, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 192685, "cached_input_tokens": 0, "completion_tokens": 3772, "thoughts_tokens": 5423, "total_tokens": 201880, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 323804, "cached_input_tokens": 221593, "completion_tokens": 1274, "thoughts_tokens": 5390, "total_tokens": 330468, "request_count": 12 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32478, "cached_input_tokens": 0, "completion_tokens": 285, "thoughts_tokens": 0, "total_tokens": 32763, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 6011, "cached_input_tokens": 0, "completion_tokens": 184, "thoughts_tokens": 1062, "total_tokens": 7257, "request_count": 4 } }, "agent_requests": 26, "main_agent_model_calls": 12, "tool_calls": 17, "main_agent_timed_model_calls": 12 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [124.084, 89.855, 21.544], "agent_gross_standard_cost_usd": 0.31545689, "judge_gross_standard_cost_usd": 0.0632805, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 426341, "agent_prompt_tokens": 410491, "agent_cached_input_tokens": 126420, "agent_thinking_tokens": 10552, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180075, "cached_input_tokens": 14208, "completion_tokens": 3847, "thoughts_tokens": 4813, "total_tokens": 188735, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 208085, "cached_input_tokens": 105060, "completion_tokens": 1122, "thoughts_tokens": 4556, "total_tokens": 213763, "request_count": 9 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16113, "cached_input_tokens": 7152, "completion_tokens": 144, "thoughts_tokens": 0, "total_tokens": 16257, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 6218, "cached_input_tokens": 0, "completion_tokens": 185, "thoughts_tokens": 1183, "total_tokens": 7586, "request_count": 4 } }, "agent_requests": 21, "main_agent_model_calls": 9, "tool_calls": 10, "main_agent_timed_model_calls": 9 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.45, 42.128], "agent_gross_standard_cost_usd": 0.26322485, "judge_gross_standard_cost_usd": 0.0444705, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 297897, "agent_prompt_tokens": 283831, "agent_cached_input_tokens": 16498, "agent_thinking_tokens": 10692, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 172963, "cached_input_tokens": 0, "completion_tokens": 2690, "thoughts_tokens": 5989, "total_tokens": 181642, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 108573, "cached_input_tokens": 16498, "completion_tokens": 585, "thoughts_tokens": 4476, "total_tokens": 113634, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 40, "thoughts_tokens": 227, "total_tokens": 937, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1625, "cached_input_tokens": 0, "completion_tokens": 59, "thoughts_tokens": 0, "total_tokens": 1684, "request_count": 1 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.907, 67.599], "agent_gross_standard_cost_usd": 0.2336832, "judge_gross_standard_cost_usd": 0.037836, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 350052, "agent_prompt_tokens": 335588, "agent_cached_input_tokens": 40205, "agent_thinking_tokens": 10554, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 208989, "cached_input_tokens": 0, "completion_tokens": 3086, "thoughts_tokens": 6852, "total_tokens": 218927, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 108192, "cached_input_tokens": 40205, "completion_tokens": 579, "thoughts_tokens": 3543, "total_tokens": 112314, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 36, "thoughts_tokens": 159, "total_tokens": 865, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17737, "cached_input_tokens": 0, "completion_tokens": 209, "thoughts_tokens": 0, "total_tokens": 17946, "request_count": 2 } }, "agent_requests": 14, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.201, 25.264, 25.544], "agent_gross_standard_cost_usd": 0.28690955, "judge_gross_standard_cost_usd": 0.069888, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 418757, "agent_prompt_tokens": 405788, "agent_cached_input_tokens": 96169, "agent_thinking_tokens": 8704, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 209614, "cached_input_tokens": 0, "completion_tokens": 3460, "thoughts_tokens": 4075, "total_tokens": 217149, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 195497, "cached_input_tokens": 96169, "completion_tokens": 770, "thoughts_tokens": 4497, "total_tokens": 200764, "request_count": 8 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 35, "thoughts_tokens": 132, "total_tokens": 844, "request_count": 1 } }, "agent_requests": 15, "main_agent_model_calls": 8, "tool_calls": 7, "main_agent_timed_model_calls": 8 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [43.735, 153.216, 51.465], "agent_gross_standard_cost_usd": 0.34852613, "judge_gross_standard_cost_usd": 0.058494, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 379677, "agent_prompt_tokens": 360042, "agent_cached_input_tokens": 39515, "agent_thinking_tokens": 15012, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 201105, "cached_input_tokens": 10136, "completion_tokens": 3718, "thoughts_tokens": 9250, "total_tokens": 214073, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 158260, "cached_input_tokens": 29379, "completion_tokens": 872, "thoughts_tokens": 5638, "total_tokens": 164770, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 33, "thoughts_tokens": 124, "total_tokens": 834, "request_count": 1 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 6, "main_agent_timed_model_calls": 7 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash35_medium", "thinking_level": "medium", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": false, "judge_score": 1.0, "customer_turn_seconds": [32.09, 135.18, 0.004], "agent_gross_standard_cost_usd": 0.30376508, "judge_gross_standard_cost_usd": 0.0394035, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 345405, "agent_prompt_tokens": 324001, "agent_cached_input_tokens": 30028, "agent_thinking_tokens": 17054, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 201180, "cached_input_tokens": 10136, "completion_tokens": 3149, "thoughts_tokens": 9645, "total_tokens": 213974, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 108697, "cached_input_tokens": 19892, "completion_tokens": 845, "thoughts_tokens": 5438, "total_tokens": 114980, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 1476, "cached_input_tokens": 0, "completion_tokens": 122, "thoughts_tokens": 1971, "total_tokens": 3569, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12648, "cached_input_tokens": 0, "completion_tokens": 234, "thoughts_tokens": 0, "total_tokens": 12882, "request_count": 2 } }, "agent_requests": 16, "main_agent_model_calls": 6, "tool_calls": 4, "main_agent_timed_model_calls": 6 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash37_high", "thinking_level": "high", "reused_completed_observation": true, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": null, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [56.832, 38.75, 28.255], "agent_gross_standard_cost_usd": 0.37203729999999996, "judge_gross_standard_cost_usd": 0.07824, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 483175, "agent_prompt_tokens": 466508, "agent_cached_input_tokens": 96524, "agent_thinking_tokens": 11458, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 228204, "cached_input_tokens": 0, "completion_tokens": 4156, "thoughts_tokens": 5559, "total_tokens": 237919, "request_count": 7 }, "gemini-3.7-flash": { "prompt_tokens": 236887, "cached_input_tokens": 96524, "completion_tokens": 963, "thoughts_tokens": 5037, "total_tokens": 242887, "request_count": 9 }, "gemini-3.8-flash": { "prompt_tokens": 1417, "cached_input_tokens": 0, "completion_tokens": 90, "thoughts_tokens": 862, "total_tokens": 2369, "request_count": 2 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 9, "main_agent_timed_model_calls": 9 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.971], "agent_gross_standard_cost_usd": 0.1838015, "judge_gross_standard_cost_usd": 0.0781965, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 151563, "agent_prompt_tokens": 146288, "agent_cached_input_tokens": 19423, "agent_thinking_tokens": 3346, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68048, "cached_input_tokens": 0, "completion_tokens": 1168, "thoughts_tokens": 3346, "total_tokens": 72562, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25210, "cached_input_tokens": 0, "completion_tokens": 418, "thoughts_tokens": 0, "total_tokens": 25628, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 53030, "cached_input_tokens": 19423, "completion_tokens": 343, "thoughts_tokens": 0, "total_tokens": 53373, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [136.372], "agent_gross_standard_cost_usd": 0.5090673250000001, "judge_gross_standard_cost_usd": 0.0790725, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 218695, "agent_prompt_tokens": 199267, "agent_cached_input_tokens": 77804, "agent_thinking_tokens": 17390, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68997, "cached_input_tokens": 14205, "completion_tokens": 1141, "thoughts_tokens": 2506, "total_tokens": 72644, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 36055, "cached_input_tokens": 8827, "completion_tokens": 448, "thoughts_tokens": 0, "total_tokens": 36503, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 94215, "cached_input_tokens": 54772, "completion_tokens": 449, "thoughts_tokens": 14884, "total_tokens": 109548, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 2 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.489], "agent_gross_standard_cost_usd": 0.093811425, "judge_gross_standard_cost_usd": 0.0754605, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 189762, "agent_prompt_tokens": 186241, "agent_cached_input_tokens": 82832, "agent_thinking_tokens": 1756, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66712, "cached_input_tokens": 14205, "completion_tokens": 1046, "thoughts_tokens": 1756, "total_tokens": 69514, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25164, "cached_input_tokens": 11077, "completion_tokens": 403, "thoughts_tokens": 0, "total_tokens": 25567, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 94365, "cached_input_tokens": 57550, "completion_tokens": 316, "thoughts_tokens": 0, "total_tokens": 94681, "request_count": 4 } }, "agent_requests": 10, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.891], "agent_gross_standard_cost_usd": 0.121538325, "judge_gross_standard_cost_usd": 0.0784095, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 178480, "agent_prompt_tokens": 171064, "agent_cached_input_tokens": 65244, "agent_thinking_tokens": 5581, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68239, "cached_input_tokens": 14205, "completion_tokens": 1101, "thoughts_tokens": 1806, "total_tokens": 71146, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25158, "cached_input_tokens": 11077, "completion_tokens": 422, "thoughts_tokens": 0, "total_tokens": 25580, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 77667, "cached_input_tokens": 39962, "completion_tokens": 312, "thoughts_tokens": 3775, "total_tokens": 81754, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.697], "agent_gross_standard_cost_usd": 0.07299311, "judge_gross_standard_cost_usd": 0.0715185, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 139563, "agent_prompt_tokens": 135867, "agent_cached_input_tokens": 55760, "agent_thinking_tokens": 1926, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66470, "cached_input_tokens": 30542, "completion_tokens": 1086, "thoughts_tokens": 1926, "total_tokens": 69482, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25166, "cached_input_tokens": 11078, "completion_tokens": 403, "thoughts_tokens": 0, "total_tokens": 25569, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 44231, "cached_input_tokens": 14140, "completion_tokens": 281, "thoughts_tokens": 0, "total_tokens": 44512, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.63], "agent_gross_standard_cost_usd": 0.10158829999999999, "judge_gross_standard_cost_usd": 0.074451, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140500, "agent_prompt_tokens": 136049, "agent_cached_input_tokens": 25283, "agent_thinking_tokens": 2699, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66464, "cached_input_tokens": 14205, "completion_tokens": 1071, "thoughts_tokens": 2108, "total_tokens": 69643, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25176, "cached_input_tokens": 11078, "completion_tokens": 403, "thoughts_tokens": 0, "total_tokens": 25579, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 44409, "cached_input_tokens": 0, "completion_tokens": 278, "thoughts_tokens": 591, "total_tokens": 45278, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [34.952], "agent_gross_standard_cost_usd": 0.09203885, "judge_gross_standard_cost_usd": 0.071415, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 164178, "agent_prompt_tokens": 158813, "agent_cached_input_tokens": 62796, "agent_thinking_tokens": 3568, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66511, "cached_input_tokens": 14205, "completion_tokens": 1057, "thoughts_tokens": 2501, "total_tokens": 70069, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25166, "cached_input_tokens": 11078, "completion_tokens": 413, "thoughts_tokens": 0, "total_tokens": 25579, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 67136, "cached_input_tokens": 37513, "completion_tokens": 327, "thoughts_tokens": 1067, "total_tokens": 68530, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-01", "suite": "Conversation ownership", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [111.885], "agent_gross_standard_cost_usd": 0.1818341, "judge_gross_standard_cost_usd": 0.0512565, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 182020, "agent_prompt_tokens": 174240, "agent_cached_input_tokens": 37738, "agent_thinking_tokens": 5958, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63640, "cached_input_tokens": 14205, "completion_tokens": 1176, "thoughts_tokens": 2561, "total_tokens": 67377, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 9108, "cached_input_tokens": 0, "completion_tokens": 253, "thoughts_tokens": 0, "total_tokens": 9361, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 101492, "cached_input_tokens": 23533, "completion_tokens": 393, "thoughts_tokens": 3397, "total_tokens": 105282, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [84.513], "agent_gross_standard_cost_usd": 0.372310925, "judge_gross_standard_cost_usd": 0.1623375, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 287999, "agent_prompt_tokens": 277851, "agent_cached_input_tokens": 83401, "agent_thinking_tokens": 6875, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 81139, "cached_input_tokens": 0, "completion_tokens": 1281, "thoughts_tokens": 1665, "total_tokens": 84085, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 105914, "cached_input_tokens": 57114, "completion_tokens": 541, "thoughts_tokens": 5210, "total_tokens": 111665, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 90798, "cached_input_tokens": 26287, "completion_tokens": 1451, "thoughts_tokens": 0, "total_tokens": 92249, "request_count": 11 } }, "agent_requests": 16, "main_agent_model_calls": 3, "tool_calls": 21, "main_agent_timed_model_calls": 3 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [46.479], "agent_gross_standard_cost_usd": 0.11629016, "judge_gross_standard_cost_usd": 0.0887295, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 196596, "agent_prompt_tokens": 190208, "agent_cached_input_tokens": 76536, "agent_thinking_tokens": 4222, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67770, "cached_input_tokens": 14197, "completion_tokens": 1291, "thoughts_tokens": 1630, "total_tokens": 70691, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 81044, "cached_input_tokens": 42943, "completion_tokens": 360, "thoughts_tokens": 2592, "total_tokens": 83996, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 41394, "cached_input_tokens": 19396, "completion_tokens": 515, "thoughts_tokens": 0, "total_tokens": 41909, "request_count": 5 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [65.974], "agent_gross_standard_cost_usd": 0.15981468499999998, "judge_gross_standard_cost_usd": 0.1341375, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 304937, "agent_prompt_tokens": 296900, "agent_cached_input_tokens": 134016, "agent_thinking_tokens": 5177, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 75568, "cached_input_tokens": 14197, "completion_tokens": 1212, "thoughts_tokens": 1466, "total_tokens": 78246, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 155413, "cached_input_tokens": 107584, "completion_tokens": 395, "thoughts_tokens": 3711, "total_tokens": 159519, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 65919, "cached_input_tokens": 12235, "completion_tokens": 1253, "thoughts_tokens": 0, "total_tokens": 67172, "request_count": 8 } }, "agent_requests": 15, "main_agent_model_calls": 5, "tool_calls": 15, "main_agent_timed_model_calls": 5 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [61.727], "agent_gross_standard_cost_usd": 0.08221296, "judge_gross_standard_cost_usd": 0.056703, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 178928, "agent_prompt_tokens": 173566, "agent_cached_input_tokens": 59074, "agent_thinking_tokens": 3205, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57197, "cached_input_tokens": 14197, "completion_tokens": 1108, "thoughts_tokens": 3205, "total_tokens": 61510, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 63082, "cached_input_tokens": 17632, "completion_tokens": 786, "thoughts_tokens": 0, "total_tokens": 63868, "request_count": 9 }, "gemini-3.7-flash": { "prompt_tokens": 53287, "cached_input_tokens": 27245, "completion_tokens": 263, "thoughts_tokens": 0, "total_tokens": 53550, "request_count": 3 } }, "agent_requests": 14, "main_agent_model_calls": 3, "tool_calls": 12, "main_agent_timed_model_calls": 3 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.017], "agent_gross_standard_cost_usd": 0.1436813, "judge_gross_standard_cost_usd": 0.0841815, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 183899, "agent_prompt_tokens": 180196, "agent_cached_input_tokens": 14276, "agent_thinking_tokens": 1727, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 67483, "cached_input_tokens": 0, "completion_tokens": 1161, "thoughts_tokens": 980, "total_tokens": 69624, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 80433, "cached_input_tokens": 14276, "completion_tokens": 302, "thoughts_tokens": 747, "total_tokens": 81482, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32280, "cached_input_tokens": 0, "completion_tokens": 513, "thoughts_tokens": 0, "total_tokens": 32793, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.053], "agent_gross_standard_cost_usd": 0.104809535, "judge_gross_standard_cost_usd": 0.0703245, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 192922, "agent_prompt_tokens": 187411, "agent_cached_input_tokens": 72695, "agent_thinking_tokens": 3532, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59963, "cached_input_tokens": 14197, "completion_tokens": 1063, "thoughts_tokens": 3069, "total_tokens": 64095, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 75199, "cached_input_tokens": 35863, "completion_tokens": 324, "thoughts_tokens": 463, "total_tokens": 75986, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 52249, "cached_input_tokens": 22635, "completion_tokens": 592, "thoughts_tokens": 0, "total_tokens": 52841, "request_count": 6 } }, "agent_requests": 12, "main_agent_model_calls": 4, "tool_calls": 9, "main_agent_timed_model_calls": 4 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [196.135], "agent_gross_standard_cost_usd": 0.188796885, "judge_gross_standard_cost_usd": 0.078162, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 254230, "agent_prompt_tokens": 244757, "agent_cached_input_tokens": 85962, "agent_thinking_tokens": 7422, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 65779, "cached_input_tokens": 14197, "completion_tokens": 1008, "thoughts_tokens": 2643, "total_tokens": 69430, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 49146, "cached_input_tokens": 5337, "completion_tokens": 638, "thoughts_tokens": 0, "total_tokens": 49784, "request_count": 7 }, "gemini-3.5-flash": { "prompt_tokens": 129832, "cached_input_tokens": 66428, "completion_tokens": 405, "thoughts_tokens": 4779, "total_tokens": 135016, "request_count": 5 } }, "agent_requests": 14, "main_agent_model_calls": 5, "tool_calls": 10, "main_agent_timed_model_calls": 5 }, { "task": "task-02", "suite": "Conversation ownership", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [57.44], "agent_gross_standard_cost_usd": 0.28248443500000003, "judge_gross_standard_cost_usd": 0.1557945, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 361422, "agent_prompt_tokens": 354431, "agent_cached_input_tokens": 166735, "agent_thinking_tokens": 2861, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 96871, "cached_input_tokens": 14197, "completion_tokens": 1898, "thoughts_tokens": 2861, "total_tokens": 101630, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 104844, "cached_input_tokens": 48883, "completion_tokens": 1701, "thoughts_tokens": 0, "total_tokens": 106545, "request_count": 13 }, "gemini-4-argon": { "prompt_tokens": 152716, "cached_input_tokens": 103655, "completion_tokens": 531, "thoughts_tokens": 0, "total_tokens": 153247, "request_count": 4 } }, "agent_requests": 20, "main_agent_model_calls": 4, "tool_calls": 21, "main_agent_timed_model_calls": 4 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [68.0], "agent_gross_standard_cost_usd": 0.180131425, "judge_gross_standard_cost_usd": 0.0757215, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 274668, "agent_prompt_tokens": 264611, "agent_cached_input_tokens": 70851, "agent_thinking_tokens": 7779, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 109823, "cached_input_tokens": 0, "completion_tokens": 1255, "thoughts_tokens": 4581, "total_tokens": 115659, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 119244, "cached_input_tokens": 60316, "completion_tokens": 520, "thoughts_tokens": 3198, "total_tokens": 122962, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 35544, "cached_input_tokens": 10535, "completion_tokens": 503, "thoughts_tokens": 0, "total_tokens": 36047, "request_count": 5 } }, "agent_requests": 13, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [88.287], "agent_gross_standard_cost_usd": 0.1638719, "judge_gross_standard_cost_usd": 0.056355, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 199794, "agent_prompt_tokens": 186567, "agent_cached_input_tokens": 68261, "agent_thinking_tokens": 11415, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61595, "cached_input_tokens": 0, "completion_tokens": 1111, "thoughts_tokens": 3144, "total_tokens": 65850, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 104337, "cached_input_tokens": 68261, "completion_tokens": 360, "thoughts_tokens": 8271, "total_tokens": 112968, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20635, "cached_input_tokens": 0, "completion_tokens": 341, "thoughts_tokens": 0, "total_tokens": 20976, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 9, "main_agent_timed_model_calls": 4 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 4.0, "customer_turn_seconds": [38.169], "agent_gross_standard_cost_usd": 0.135029175, "judge_gross_standard_cost_usd": 0.0617505, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 207428, "agent_prompt_tokens": 204134, "agent_cached_input_tokens": 65102, "agent_thinking_tokens": 1620, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60810, "cached_input_tokens": 0, "completion_tokens": 946, "thoughts_tokens": 1620, "total_tokens": 63376, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 122688, "cached_input_tokens": 59831, "completion_tokens": 341, "thoughts_tokens": 0, "total_tokens": 123029, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20636, "cached_input_tokens": 5271, "completion_tokens": 387, "thoughts_tokens": 0, "total_tokens": 21023, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 5, "tool_calls": 8, "main_agent_timed_model_calls": 5 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.433], "agent_gross_standard_cost_usd": 0.089325125, "judge_gross_standard_cost_usd": 0.05538, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 138475, "agent_prompt_tokens": 132959, "agent_cached_input_tokens": 28098, "agent_thinking_tokens": 3940, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60336, "cached_input_tokens": 0, "completion_tokens": 957, "thoughts_tokens": 2217, "total_tokens": 63510, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 43197, "cached_input_tokens": 15845, "completion_tokens": 239, "thoughts_tokens": 1723, "total_tokens": 45159, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29426, "cached_input_tokens": 12253, "completion_tokens": 380, "thoughts_tokens": 0, "total_tokens": 29806, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.731], "agent_gross_standard_cost_usd": 0.0906976, "judge_gross_standard_cost_usd": 0.056724, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 160648, "agent_prompt_tokens": 156160, "agent_cached_input_tokens": 44243, "agent_thinking_tokens": 2945, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60323, "cached_input_tokens": 0, "completion_tokens": 917, "thoughts_tokens": 2133, "total_tokens": 63373, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 66429, "cached_input_tokens": 37253, "completion_tokens": 213, "thoughts_tokens": 812, "total_tokens": 67454, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29408, "cached_input_tokens": 6990, "completion_tokens": 413, "thoughts_tokens": 0, "total_tokens": 29821, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [66.474], "agent_gross_standard_cost_usd": 0.11868645, "judge_gross_standard_cost_usd": 0.055674, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155378, "agent_prompt_tokens": 147236, "agent_cached_input_tokens": 37338, "agent_thinking_tokens": 6414, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59835, "cached_input_tokens": 0, "completion_tokens": 1082, "thoughts_tokens": 2732, "total_tokens": 63649, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 66720, "cached_input_tokens": 37338, "completion_tokens": 300, "thoughts_tokens": 3682, "total_tokens": 70702, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20681, "cached_input_tokens": 0, "completion_tokens": 346, "thoughts_tokens": 0, "total_tokens": 21027, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [199.093], "agent_gross_standard_cost_usd": 1.31788065, "judge_gross_standard_cost_usd": 0.3132975, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 1927754, "agent_prompt_tokens": 1917046, "agent_cached_input_tokens": 1368982, "agent_thinking_tokens": 3137, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 124751, "cached_input_tokens": 0, "completion_tokens": 1040, "thoughts_tokens": 2656, "total_tokens": 128447, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 1504955, "cached_input_tokens": 1273856, "completion_tokens": 1681, "thoughts_tokens": 0, "total_tokens": 1506636, "request_count": 27 }, "gemini-3.1-flash-lite": { "prompt_tokens": 287340, "cached_input_tokens": 95126, "completion_tokens": 4850, "thoughts_tokens": 481, "total_tokens": 292671, "request_count": 39 } }, "agent_requests": 68, "main_agent_model_calls": 27, "tool_calls": 65, "main_agent_timed_model_calls": 27 }, { "task": "task-03", "suite": "Catalog and shipping evidence", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [78.868], "agent_gross_standard_cost_usd": 0.32471602499999996, "judge_gross_standard_cost_usd": 0.065082, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 230226, "agent_prompt_tokens": 220932, "agent_cached_input_tokens": 96498, "agent_thinking_tokens": 7623, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66593, "cached_input_tokens": 0, "completion_tokens": 906, "thoughts_tokens": 2752, "total_tokens": 70251, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 21150, "cached_input_tokens": 5267, "completion_tokens": 395, "thoughts_tokens": 0, "total_tokens": 21545, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 133189, "cached_input_tokens": 91231, "completion_tokens": 370, "thoughts_tokens": 4871, "total_tokens": 138430, "request_count": 4 } }, "agent_requests": 10, "main_agent_model_calls": 4, "tool_calls": 11, "main_agent_timed_model_calls": 4 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.466], "agent_gross_standard_cost_usd": 0.152159075, "judge_gross_standard_cost_usd": 0.061248, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 235053, "agent_prompt_tokens": 224804, "agent_cached_input_tokens": 104980, "agent_thinking_tokens": 8653, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62779, "cached_input_tokens": 0, "completion_tokens": 893, "thoughts_tokens": 3004, "total_tokens": 66676, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 132686, "cached_input_tokens": 92731, "completion_tokens": 290, "thoughts_tokens": 5649, "total_tokens": 138625, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29339, "cached_input_tokens": 12249, "completion_tokens": 413, "thoughts_tokens": 0, "total_tokens": 29752, "request_count": 4 } }, "agent_requests": 11, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.199], "agent_gross_standard_cost_usd": 0.08866075, "judge_gross_standard_cost_usd": 0.0563985, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 138809, "agent_prompt_tokens": 133179, "agent_cached_input_tokens": 21100, "agent_thinking_tokens": 4136, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60225, "cached_input_tokens": 0, "completion_tokens": 858, "thoughts_tokens": 2898, "total_tokens": 63981, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29791, "cached_input_tokens": 5256, "completion_tokens": 412, "thoughts_tokens": 0, "total_tokens": 30203, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 43163, "cached_input_tokens": 15844, "completion_tokens": 224, "thoughts_tokens": 1238, "total_tokens": 44625, "request_count": 2 } }, "agent_requests": 9, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [77.76], "agent_gross_standard_cost_usd": 0.17988659999999998, "judge_gross_standard_cost_usd": 0.0574515, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 231814, "agent_prompt_tokens": 222771, "agent_cached_input_tokens": 18236, "agent_thinking_tokens": 6694, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 105769, "cached_input_tokens": 0, "completion_tokens": 1450, "thoughts_tokens": 4842, "total_tokens": 112061, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 87678, "cached_input_tokens": 18236, "completion_tokens": 493, "thoughts_tokens": 1852, "total_tokens": 90023, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29324, "cached_input_tokens": 0, "completion_tokens": 406, "thoughts_tokens": 0, "total_tokens": 29730, "request_count": 4 } }, "agent_requests": 11, "main_agent_model_calls": 4, "tool_calls": 8, "main_agent_timed_model_calls": 4 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [46.711], "agent_gross_standard_cost_usd": 0.091592725, "judge_gross_standard_cost_usd": 0.0587205, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 161342, "agent_prompt_tokens": 155534, "agent_cached_input_tokens": 49498, "agent_thinking_tokens": 4019, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60208, "cached_input_tokens": 0, "completion_tokens": 1098, "thoughts_tokens": 3350, "total_tokens": 64656, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 65939, "cached_input_tokens": 37247, "completion_tokens": 285, "thoughts_tokens": 669, "total_tokens": 66893, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29387, "cached_input_tokens": 12251, "completion_tokens": 406, "thoughts_tokens": 0, "total_tokens": 29793, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [174.355], "agent_gross_standard_cost_usd": 0.22603035, "judge_gross_standard_cost_usd": 0.067755, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 327649, "agent_prompt_tokens": 316239, "agent_cached_input_tokens": 154810, "agent_thinking_tokens": 9515, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68347, "cached_input_tokens": 0, "completion_tokens": 1089, "thoughts_tokens": 3809, "total_tokens": 73245, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 227329, "cached_input_tokens": 154810, "completion_tokens": 477, "thoughts_tokens": 5706, "total_tokens": 233512, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 20563, "cached_input_tokens": 0, "completion_tokens": 329, "thoughts_tokens": 0, "total_tokens": 20892, "request_count": 3 } }, "agent_requests": 13, "main_agent_model_calls": 8, "tool_calls": 11, "main_agent_timed_model_calls": 8 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.739], "agent_gross_standard_cost_usd": 0.238871475, "judge_gross_standard_cost_usd": 0.060816, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 284199, "agent_prompt_tokens": 280571, "agent_cached_input_tokens": 157111, "agent_thinking_tokens": 1922, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63994, "cached_input_tokens": 0, "completion_tokens": 880, "thoughts_tokens": 1922, "total_tokens": 66796, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 187190, "cached_input_tokens": 144860, "completion_tokens": 462, "thoughts_tokens": 0, "total_tokens": 187652, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29387, "cached_input_tokens": 12251, "completion_tokens": 364, "thoughts_tokens": 0, "total_tokens": 29751, "request_count": 4 } }, "agent_requests": 12, "main_agent_model_calls": 6, "tool_calls": 14, "main_agent_timed_model_calls": 6 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [155.406], "agent_gross_standard_cost_usd": 0.34499715, "judge_gross_standard_cost_usd": 0.0751575, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 277337, "agent_prompt_tokens": 268145, "agent_cached_input_tokens": 130350, "agent_thinking_tokens": 7299, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 68927, "cached_input_tokens": 0, "completion_tokens": 865, "thoughts_tokens": 2273, "total_tokens": 72065, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 163716, "cached_input_tokens": 119816, "completion_tokens": 451, "thoughts_tokens": 5026, "total_tokens": 169193, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 35502, "cached_input_tokens": 10534, "completion_tokens": 577, "thoughts_tokens": 0, "total_tokens": 36079, "request_count": 5 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 14, "main_agent_timed_model_calls": 5 }, { "task": "task-04", "suite": "Catalog and shipping evidence", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [53.623], "agent_gross_standard_cost_usd": 0.11197175000000001, "judge_gross_standard_cost_usd": 0.056622, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 191704, "agent_prompt_tokens": 188256, "agent_cached_input_tokens": 61640, "agent_thinking_tokens": 1901, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61415, "cached_input_tokens": 0, "completion_tokens": 856, "thoughts_tokens": 1901, "total_tokens": 64172, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 97463, "cached_input_tokens": 49390, "completion_tokens": 252, "thoughts_tokens": 0, "total_tokens": 97715, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29378, "cached_input_tokens": 12250, "completion_tokens": 439, "thoughts_tokens": 0, "total_tokens": 29817, "request_count": 4 } }, "agent_requests": 10, "main_agent_model_calls": 4, "tool_calls": 8, "main_agent_timed_model_calls": 4 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [19.367, 23.391], "agent_gross_standard_cost_usd": 0.127973025, "judge_gross_standard_cost_usd": 0.048396, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 225216, "agent_prompt_tokens": 217860, "agent_cached_input_tokens": 39730, "agent_thinking_tokens": 4504, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 120863, "cached_input_tokens": 0, "completion_tokens": 2123, "thoughts_tokens": 4504, "total_tokens": 127490, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 22251, "cached_input_tokens": 7039, "completion_tokens": 381, "thoughts_tokens": 0, "total_tokens": 22632, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 74746, "cached_input_tokens": 32691, "completion_tokens": 348, "thoughts_tokens": 0, "total_tokens": 75094, "request_count": 4 } }, "agent_requests": 13, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [21.7, 21.41], "agent_gross_standard_cost_usd": 0.131150705, "judge_gross_standard_cost_usd": 0.0505695, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 226201, "agent_prompt_tokens": 219067, "agent_cached_input_tokens": 48187, "agent_thinking_tokens": 4135, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 121350, "cached_input_tokens": 10141, "completion_tokens": 2126, "thoughts_tokens": 3522, "total_tokens": 126998, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 22275, "cached_input_tokens": 7041, "completion_tokens": 512, "thoughts_tokens": 0, "total_tokens": 22787, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 75442, "cached_input_tokens": 31005, "completion_tokens": 361, "thoughts_tokens": 613, "total_tokens": 76416, "request_count": 4 } }, "agent_requests": 13, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [62.82, 76.575], "agent_gross_standard_cost_usd": 0.25582483, "judge_gross_standard_cost_usd": 0.0646155, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 386673, "agent_prompt_tokens": 373383, "agent_cached_input_tokens": 94258, "agent_thinking_tokens": 9934, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180932, "cached_input_tokens": 10141, "completion_tokens": 2247, "thoughts_tokens": 8308, "total_tokens": 191487, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 170941, "cached_input_tokens": 77075, "completion_tokens": 726, "thoughts_tokens": 1626, "total_tokens": 173293, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 21510, "cached_input_tokens": 7042, "completion_tokens": 383, "thoughts_tokens": 0, "total_tokens": 21893, "request_count": 4 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 8, "main_agent_timed_model_calls": 8 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [50.472, 52.846], "agent_gross_standard_cost_usd": 0.312439075, "judge_gross_standard_cost_usd": 0.1140225, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 429822, "agent_prompt_tokens": 418016, "agent_cached_input_tokens": 97307, "agent_thinking_tokens": 7893, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 145918, "cached_input_tokens": 0, "completion_tokens": 2448, "thoughts_tokens": 3939, "total_tokens": 152305, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 205923, "cached_input_tokens": 82948, "completion_tokens": 626, "thoughts_tokens": 3954, "total_tokens": 210503, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 66175, "cached_input_tokens": 14359, "completion_tokens": 839, "thoughts_tokens": 0, "total_tokens": 67014, "request_count": 8 } }, "agent_requests": 20, "main_agent_model_calls": 8, "tool_calls": 16, "main_agent_timed_model_calls": 8 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.884, 37.004], "agent_gross_standard_cost_usd": 0.439070825, "judge_gross_standard_cost_usd": 0.124935, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 445046, "agent_prompt_tokens": 436354, "agent_cached_input_tokens": 166854, "agent_thinking_tokens": 4916, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 147073, "cached_input_tokens": 10135, "completion_tokens": 2194, "thoughts_tokens": 4916, "total_tokens": 154183, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 82284, "cached_input_tokens": 30621, "completion_tokens": 769, "thoughts_tokens": 0, "total_tokens": 83053, "request_count": 10 }, "gemini-4-argon": { "prompt_tokens": 206997, "cached_input_tokens": 126098, "completion_tokens": 813, "thoughts_tokens": 0, "total_tokens": 207810, "request_count": 6 } }, "agent_requests": 20, "main_agent_model_calls": 6, "tool_calls": 21, "main_agent_timed_model_calls": 6 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [84.963, 160.322], "agent_gross_standard_cost_usd": 0.831115825, "judge_gross_standard_cost_usd": 0.133851, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 568329, "agent_prompt_tokens": 548832, "agent_cached_input_tokens": 231274, "agent_thinking_tokens": 15934, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 149371, "cached_input_tokens": 0, "completion_tokens": 1933, "thoughts_tokens": 4510, "total_tokens": 155814, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 333193, "cached_input_tokens": 216611, "completion_tokens": 884, "thoughts_tokens": 11424, "total_tokens": 345501, "request_count": 9 }, "gemini-3.1-flash-lite": { "prompt_tokens": 66268, "cached_input_tokens": 14663, "completion_tokens": 746, "thoughts_tokens": 0, "total_tokens": 67014, "request_count": 8 } }, "agent_requests": 21, "main_agent_model_calls": 9, "tool_calls": 22, "main_agent_timed_model_calls": 9 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [27.174, 16.405], "agent_gross_standard_cost_usd": 0.159179645, "judge_gross_standard_cost_usd": 0.0805935, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 306036, "agent_prompt_tokens": 300267, "agent_cached_input_tokens": 86795, "agent_thinking_tokens": 2667, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 133508, "cached_input_tokens": 10139, "completion_tokens": 2121, "thoughts_tokens": 2667, "total_tokens": 138296, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 109424, "cached_input_tokens": 49295, "completion_tokens": 443, "thoughts_tokens": 0, "total_tokens": 109867, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 57335, "cached_input_tokens": 27361, "completion_tokens": 538, "thoughts_tokens": 0, "total_tokens": 57873, "request_count": 6 } }, "agent_requests": 15, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-05", "suite": "Catalog and shipping evidence", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.015, 91.483], "agent_gross_standard_cost_usd": 0.27565977, "judge_gross_standard_cost_usd": 0.0817905, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 424920, "agent_prompt_tokens": 410435, "agent_cached_input_tokens": 166523, "agent_thinking_tokens": 11688, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 134359, "cached_input_tokens": 10139, "completion_tokens": 1809, "thoughts_tokens": 5115, "total_tokens": 141283, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 35697, "cached_input_tokens": 7278, "completion_tokens": 415, "thoughts_tokens": 0, "total_tokens": 36112, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 240379, "cached_input_tokens": 149106, "completion_tokens": 573, "thoughts_tokens": 6573, "total_tokens": 247525, "request_count": 9 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 13, "main_agent_timed_model_calls": 9 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.214], "agent_gross_standard_cost_usd": 0.0072784, "judge_gross_standard_cost_usd": 0.008676, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16510, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 594, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 463, "thoughts_tokens": 594, "total_tokens": 16510, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.01], "agent_gross_standard_cost_usd": 0.0072218000000000004, "judge_gross_standard_cost_usd": 0.009096, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16490, "agent_prompt_tokens": 15456, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 559, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15456, "cached_input_tokens": 0, "completion_tokens": 475, "thoughts_tokens": 559, "total_tokens": 16490, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.841], "agent_gross_standard_cost_usd": 0.0081909, "judge_gross_standard_cost_usd": 0.0095415, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16875, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 948, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 474, "thoughts_tokens": 948, "total_tokens": 16875, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.153], "agent_gross_standard_cost_usd": 0.0075318, "judge_gross_standard_cost_usd": 0.0098115, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16614, "agent_prompt_tokens": 15456, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 681, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15456, "cached_input_tokens": 0, "completion_tokens": 477, "thoughts_tokens": 681, "total_tokens": 16614, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.013], "agent_gross_standard_cost_usd": 0.0073034, "judge_gross_standard_cost_usd": 0.009912, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16520, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 603, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 464, "thoughts_tokens": 603, "total_tokens": 16520, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.284], "agent_gross_standard_cost_usd": 0.0073143, "judge_gross_standard_cost_usd": 0.009501, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16527, "agent_prompt_tokens": 15456, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 598, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15456, "cached_input_tokens": 0, "completion_tokens": 473, "thoughts_tokens": 598, "total_tokens": 16527, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [3.898], "agent_gross_standard_cost_usd": 0.0074605999999999995, "judge_gross_standard_cost_usd": 0.009105, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16582, "agent_prompt_tokens": 15452, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 655, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15452, "cached_input_tokens": 0, "completion_tokens": 475, "thoughts_tokens": 655, "total_tokens": 16582, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-06", "suite": "Conversation ownership", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [4.012], "agent_gross_standard_cost_usd": 0.0073758999999999995, "judge_gross_standard_cost_usd": 0.0094845, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 16549, "agent_prompt_tokens": 15453, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 609, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 15453, "cached_input_tokens": 0, "completion_tokens": 487, "thoughts_tokens": 609, "total_tokens": 16549, "request_count": 1 } }, "agent_requests": 1, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.057], "agent_gross_standard_cost_usd": 0.0694899, "judge_gross_standard_cost_usd": 0.026073, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 97928, "agent_prompt_tokens": 94000, "agent_cached_input_tokens": 14510, "agent_thinking_tokens": 3002, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54593, "cached_input_tokens": 0, "completion_tokens": 718, "thoughts_tokens": 2630, "total_tokens": 57941, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 39407, "cached_input_tokens": 14510, "completion_tokens": 208, "thoughts_tokens": 372, "total_tokens": 39987, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [52.432], "agent_gross_standard_cost_usd": 0.1173106, "judge_gross_standard_cost_usd": 0.026967, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 124010, "agent_prompt_tokens": 119170, "agent_cached_input_tokens": 14530, "agent_thinking_tokens": 3745, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54577, "cached_input_tokens": 0, "completion_tokens": 886, "thoughts_tokens": 2175, "total_tokens": 57638, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 64593, "cached_input_tokens": 14530, "completion_tokens": 209, "thoughts_tokens": 1570, "total_tokens": 66372, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.578], "agent_gross_standard_cost_usd": 0.2717362, "judge_gross_standard_cost_usd": 0.034377, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 192026, "agent_prompt_tokens": 188719, "agent_cached_input_tokens": 72300, "agent_thinking_tokens": 2134, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59779, "cached_input_tokens": 0, "completion_tokens": 915, "thoughts_tokens": 2134, "total_tokens": 62828, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 128940, "cached_input_tokens": 72300, "completion_tokens": 258, "thoughts_tokens": 0, "total_tokens": 129198, "request_count": 5 } }, "agent_requests": 7, "main_agent_model_calls": 5, "tool_calls": 6, "main_agent_timed_model_calls": 5 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.572], "agent_gross_standard_cost_usd": 0.2370985, "judge_gross_standard_cost_usd": 0.036792, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 184788, "agent_prompt_tokens": 179461, "agent_cached_input_tokens": 81382, "agent_thinking_tokens": 4004, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60747, "cached_input_tokens": 0, "completion_tokens": 1010, "thoughts_tokens": 2034, "total_tokens": 63791, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 118714, "cached_input_tokens": 81382, "completion_tokens": 313, "thoughts_tokens": 1970, "total_tokens": 120997, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [21.22], "agent_gross_standard_cost_usd": 0.07013655, "judge_gross_standard_cost_usd": 0.02544, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 97716, "agent_prompt_tokens": 93905, "agent_cached_input_tokens": 14451, "agent_thinking_tokens": 2922, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54583, "cached_input_tokens": 0, "completion_tokens": 684, "thoughts_tokens": 2175, "total_tokens": 57442, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 39322, "cached_input_tokens": 14451, "completion_tokens": 205, "thoughts_tokens": 747, "total_tokens": 40274, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.623], "agent_gross_standard_cost_usd": 0.11989625000000001, "judge_gross_standard_cost_usd": 0.0344175, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 163371, "agent_prompt_tokens": 155469, "agent_cached_input_tokens": 62107, "agent_thinking_tokens": 6816, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59719, "cached_input_tokens": 0, "completion_tokens": 830, "thoughts_tokens": 2583, "total_tokens": 63132, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 95750, "cached_input_tokens": 62107, "completion_tokens": 256, "thoughts_tokens": 4233, "total_tokens": 100239, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [21.584], "agent_gross_standard_cost_usd": 0.06886329999999999, "judge_gross_standard_cost_usd": 0.0252165, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 97230, "agent_prompt_tokens": 93949, "agent_cached_input_tokens": 14454, "agent_thinking_tokens": 2390, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54579, "cached_input_tokens": 0, "completion_tokens": 691, "thoughts_tokens": 1641, "total_tokens": 56911, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 39370, "cached_input_tokens": 14454, "completion_tokens": 200, "thoughts_tokens": 749, "total_tokens": 40319, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-07", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [51.704], "agent_gross_standard_cost_usd": 0.07552212, "judge_gross_standard_cost_usd": 0.0263775, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122116, "agent_prompt_tokens": 118360, "agent_cached_input_tokens": 50368, "agent_thinking_tokens": 2862, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54572, "cached_input_tokens": 14204, "completion_tokens": 697, "thoughts_tokens": 1712, "total_tokens": 56981, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 63788, "cached_input_tokens": 36164, "completion_tokens": 197, "thoughts_tokens": 1150, "total_tokens": 65135, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [166.465], "agent_gross_standard_cost_usd": 0.1399654, "judge_gross_standard_cost_usd": 0.023274, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 154253, "agent_prompt_tokens": 141940, "agent_cached_input_tokens": 28705, "agent_thinking_tokens": 10289, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 81898, "cached_input_tokens": 0, "completion_tokens": 1439, "thoughts_tokens": 5555, "total_tokens": 88892, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 59391, "cached_input_tokens": 28705, "completion_tokens": 524, "thoughts_tokens": 4734, "total_tokens": 64649, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 651, "cached_input_tokens": 0, "completion_tokens": 61, "thoughts_tokens": 0, "total_tokens": 712, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.534], "agent_gross_standard_cost_usd": 0.22385719999999998, "judge_gross_standard_cost_usd": 0.029457, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 175888, "agent_prompt_tokens": 173042, "agent_cached_input_tokens": 74205, "agent_thinking_tokens": 1836, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 53664, "cached_input_tokens": 0, "completion_tokens": 718, "thoughts_tokens": 1836, "total_tokens": 56218, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 119378, "cached_input_tokens": 74205, "completion_tokens": 292, "thoughts_tokens": 0, "total_tokens": 119670, "request_count": 6 } }, "agent_requests": 8, "main_agent_model_calls": 6, "tool_calls": 6, "main_agent_timed_model_calls": 6 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [117.999], "agent_gross_standard_cost_usd": 0.402034, "judge_gross_standard_cost_usd": 0.028221, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 181479, "agent_prompt_tokens": 166264, "agent_cached_input_tokens": 82259, "agent_thinking_tokens": 13811, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 53869, "cached_input_tokens": 0, "completion_tokens": 1088, "thoughts_tokens": 2079, "total_tokens": 57036, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 112395, "cached_input_tokens": 82259, "completion_tokens": 316, "thoughts_tokens": 11732, "total_tokens": 124443, "request_count": 5 } }, "agent_requests": 7, "main_agent_model_calls": 5, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [31.622], "agent_gross_standard_cost_usd": 0.08318955, "judge_gross_standard_cost_usd": 0.0251715, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109802, "agent_prompt_tokens": 104656, "agent_cached_input_tokens": 25311, "agent_thinking_tokens": 4000, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52608, "cached_input_tokens": 0, "completion_tokens": 905, "thoughts_tokens": 2113, "total_tokens": 55626, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 52048, "cached_input_tokens": 25311, "completion_tokens": 241, "thoughts_tokens": 1887, "total_tokens": 54176, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [57.115], "agent_gross_standard_cost_usd": 0.11683705, "judge_gross_standard_cost_usd": 0.032085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 114789, "agent_prompt_tokens": 104885, "agent_cached_input_tokens": 25303, "agent_thinking_tokens": 8689, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52822, "cached_input_tokens": 0, "completion_tokens": 915, "thoughts_tokens": 2530, "total_tokens": 56267, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 52063, "cached_input_tokens": 25303, "completion_tokens": 300, "thoughts_tokens": 6159, "total_tokens": 58522, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [55.08], "agent_gross_standard_cost_usd": 0.0909031, "judge_gross_standard_cost_usd": 0.026046, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109361, "agent_prompt_tokens": 104284, "agent_cached_input_tokens": 14616, "agent_thinking_tokens": 3996, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52549, "cached_input_tokens": 0, "completion_tokens": 879, "thoughts_tokens": 3283, "total_tokens": 56711, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 51735, "cached_input_tokens": 14616, "completion_tokens": 202, "thoughts_tokens": 713, "total_tokens": 52650, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [21.785], "agent_gross_standard_cost_usd": 0.0599875, "judge_gross_standard_cost_usd": 0.02238, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 82964, "agent_prompt_tokens": 79110, "agent_cached_input_tokens": 12474, "agent_thinking_tokens": 2928, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48348, "cached_input_tokens": 0, "completion_tokens": 723, "thoughts_tokens": 1822, "total_tokens": 50893, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 30762, "cached_input_tokens": 12474, "completion_tokens": 203, "thoughts_tokens": 1106, "total_tokens": 32071, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-08", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.666], "agent_gross_standard_cost_usd": 0.06657, "judge_gross_standard_cost_usd": 0.021996, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 101128, "agent_prompt_tokens": 96212, "agent_cached_input_tokens": 28684, "agent_thinking_tokens": 3905, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48363, "cached_input_tokens": 0, "completion_tokens": 781, "thoughts_tokens": 3101, "total_tokens": 52245, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 47849, "cached_input_tokens": 28684, "completion_tokens": 230, "thoughts_tokens": 804, "total_tokens": 48883, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.685], "agent_gross_standard_cost_usd": 0.2525151, "judge_gross_standard_cost_usd": 0.0322665, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 179037, "agent_prompt_tokens": 176229, "agent_cached_input_tokens": 68255, "agent_thinking_tokens": 1671, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55312, "cached_input_tokens": 0, "completion_tokens": 874, "thoughts_tokens": 1671, "total_tokens": 57857, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 120917, "cached_input_tokens": 68255, "completion_tokens": 263, "thoughts_tokens": 0, "total_tokens": 121180, "request_count": 5 } }, "agent_requests": 7, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [43.787], "agent_gross_standard_cost_usd": 0.2117284, "judge_gross_standard_cost_usd": 0.0311955, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 136392, "agent_prompt_tokens": 130420, "agent_cached_input_tokens": 44447, "agent_thinking_tokens": 4805, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55240, "cached_input_tokens": 0, "completion_tokens": 914, "thoughts_tokens": 2292, "total_tokens": 58446, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 75180, "cached_input_tokens": 44447, "completion_tokens": 253, "thoughts_tokens": 2513, "total_tokens": 77946, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.559], "agent_gross_standard_cost_usd": 0.08076755, "judge_gross_standard_cost_usd": 0.0251235, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116485, "agent_prompt_tokens": 112054, "agent_cached_input_tokens": 32389, "agent_thinking_tokens": 3336, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50784, "cached_input_tokens": 0, "completion_tokens": 875, "thoughts_tokens": 2301, "total_tokens": 53960, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 61270, "cached_input_tokens": 32389, "completion_tokens": 220, "thoughts_tokens": 1035, "total_tokens": 62525, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [64.838], "agent_gross_standard_cost_usd": 0.10749195, "judge_gross_standard_cost_usd": 0.0320415, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 149920, "agent_prompt_tokens": 143934, "agent_cached_input_tokens": 54383, "agent_thinking_tokens": 4840, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55035, "cached_input_tokens": 0, "completion_tokens": 907, "thoughts_tokens": 1862, "total_tokens": 57804, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 88899, "cached_input_tokens": 54383, "completion_tokens": 239, "thoughts_tokens": 2978, "total_tokens": 92116, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.914], "agent_gross_standard_cost_usd": 0.0774656, "judge_gross_standard_cost_usd": 0.024561, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116142, "agent_prompt_tokens": 112381, "agent_cached_input_tokens": 32400, "agent_thinking_tokens": 2685, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50957, "cached_input_tokens": 0, "completion_tokens": 871, "thoughts_tokens": 2014, "total_tokens": 53842, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 61424, "cached_input_tokens": 32400, "completion_tokens": 205, "thoughts_tokens": 671, "total_tokens": 62300, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [86.166], "agent_gross_standard_cost_usd": 0.0799194, "judge_gross_standard_cost_usd": 0.0256875, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116756, "agent_prompt_tokens": 112355, "agent_cached_input_tokens": 32392, "agent_thinking_tokens": 3105, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50997, "cached_input_tokens": 0, "completion_tokens": 1085, "thoughts_tokens": 2254, "total_tokens": 54336, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 61358, "cached_input_tokens": 32392, "completion_tokens": 211, "thoughts_tokens": 851, "total_tokens": 62420, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [22.56], "agent_gross_standard_cost_usd": 0.077582, "judge_gross_standard_cost_usd": 0.0249075, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 117243, "agent_prompt_tokens": 112348, "agent_cached_input_tokens": 36130, "agent_thinking_tokens": 3694, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50990, "cached_input_tokens": 0, "completion_tokens": 962, "thoughts_tokens": 2889, "total_tokens": 54841, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 61358, "cached_input_tokens": 36130, "completion_tokens": 239, "thoughts_tokens": 805, "total_tokens": 62402, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-09", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [193.907], "agent_gross_standard_cost_usd": 0.1016463, "judge_gross_standard_cost_usd": 0.032154, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 150140, "agent_prompt_tokens": 144689, "agent_cached_input_tokens": 58236, "agent_thinking_tokens": 4230, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 55273, "cached_input_tokens": 0, "completion_tokens": 949, "thoughts_tokens": 2051, "total_tokens": 58273, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 89416, "cached_input_tokens": 58236, "completion_tokens": 272, "thoughts_tokens": 2179, "total_tokens": 91867, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [74.837], "agent_gross_standard_cost_usd": 0.36796799999999996, "judge_gross_standard_cost_usd": 0.024198, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 161414, "agent_prompt_tokens": 151824, "agent_cached_input_tokens": 15965, "agent_thinking_tokens": 7876, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77255, "cached_input_tokens": 0, "completion_tokens": 1255, "thoughts_tokens": 3305, "total_tokens": 81815, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 73887, "cached_input_tokens": 15965, "completion_tokens": 410, "thoughts_tokens": 4371, "total_tokens": 78668, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 49, "thoughts_tokens": 200, "total_tokens": 931, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [82.262], "agent_gross_standard_cost_usd": 0.122252, "judge_gross_standard_cost_usd": 0.020733, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 168202, "agent_prompt_tokens": 161509, "agent_cached_input_tokens": 40252, "agent_thinking_tokens": 5207, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77149, "cached_input_tokens": 0, "completion_tokens": 1147, "thoughts_tokens": 3511, "total_tokens": 81807, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 84360, "cached_input_tokens": 40252, "completion_tokens": 339, "thoughts_tokens": 1696, "total_tokens": 86395, "request_count": 6 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [63.984], "agent_gross_standard_cost_usd": 0.12937065, "judge_gross_standard_cost_usd": 0.0211905, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140922, "agent_prompt_tokens": 130628, "agent_cached_input_tokens": 25825, "agent_thinking_tokens": 8942, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77148, "cached_input_tokens": 0, "completion_tokens": 1032, "thoughts_tokens": 2235, "total_tokens": 80415, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 53480, "cached_input_tokens": 25825, "completion_tokens": 320, "thoughts_tokens": 6707, "total_tokens": 60507, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [97.565], "agent_gross_standard_cost_usd": 0.10535939999999999, "judge_gross_standard_cost_usd": 0.021546, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 132829, "agent_prompt_tokens": 127819, "agent_cached_input_tokens": 10990, "agent_thinking_tokens": 3604, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77148, "cached_input_tokens": 0, "completion_tokens": 1094, "thoughts_tokens": 2212, "total_tokens": 80454, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 49989, "cached_input_tokens": 10990, "completion_tokens": 264, "thoughts_tokens": 1203, "total_tokens": 51456, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 48, "thoughts_tokens": 189, "total_tokens": 919, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.086], "agent_gross_standard_cost_usd": 0.08408105, "judge_gross_standard_cost_usd": 0.020742, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116545, "agent_prompt_tokens": 110079, "agent_cached_input_tokens": 10991, "agent_thinking_tokens": 4966, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77158, "cached_input_tokens": 0, "completion_tokens": 1152, "thoughts_tokens": 3269, "total_tokens": 81579, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 32239, "cached_input_tokens": 10991, "completion_tokens": 274, "thoughts_tokens": 1496, "total_tokens": 34009, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 74, "thoughts_tokens": 201, "total_tokens": 957, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.222], "agent_gross_standard_cost_usd": 0.0794299, "judge_gross_standard_cost_usd": 0.0220875, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 132756, "agent_prompt_tokens": 127573, "agent_cached_input_tokens": 29424, "agent_thinking_tokens": 3746, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77156, "cached_input_tokens": 0, "completion_tokens": 1082, "thoughts_tokens": 2909, "total_tokens": 81147, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 49735, "cached_input_tokens": 29424, "completion_tokens": 305, "thoughts_tokens": 670, "total_tokens": 50710, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 167, "total_tokens": 899, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [41.324], "agent_gross_standard_cost_usd": 0.10631175, "judge_gross_standard_cost_usd": 0.022737, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 136225, "agent_prompt_tokens": 128225, "agent_cached_input_tokens": 29491, "agent_thinking_tokens": 6425, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 77187, "cached_input_tokens": 0, "completion_tokens": 1208, "thoughts_tokens": 2644, "total_tokens": 81039, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 50356, "cached_input_tokens": 29491, "completion_tokens": 305, "thoughts_tokens": 3476, "total_tokens": 54137, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 62, "thoughts_tokens": 305, "total_tokens": 1049, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-10", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [46.868], "agent_gross_standard_cost_usd": 0.262820075, "judge_gross_standard_cost_usd": 0.0799815, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 261660, "agent_prompt_tokens": 257235, "agent_cached_input_tokens": 79790, "agent_thinking_tokens": 2282, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 87820, "cached_input_tokens": 0, "completion_tokens": 1330, "thoughts_tokens": 2089, "total_tokens": 91239, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 112423, "cached_input_tokens": 64549, "completion_tokens": 473, "thoughts_tokens": 0, "total_tokens": 112896, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 682, "cached_input_tokens": 0, "completion_tokens": 72, "thoughts_tokens": 193, "total_tokens": 947, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 56310, "cached_input_tokens": 15241, "completion_tokens": 268, "thoughts_tokens": 0, "total_tokens": 56578, "request_count": 4 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 10, "main_agent_timed_model_calls": 5 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [95.067], "agent_gross_standard_cost_usd": 0.25287745, "judge_gross_standard_cost_usd": 0.455799, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 576292, "agent_prompt_tokens": 567104, "agent_cached_input_tokens": 170594, "agent_thinking_tokens": 5322, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 149417, "cached_input_tokens": 0, "completion_tokens": 1213, "thoughts_tokens": 2801, "total_tokens": 153431, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 183401, "cached_input_tokens": 109240, "completion_tokens": 456, "thoughts_tokens": 2521, "total_tokens": 186378, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 234286, "cached_input_tokens": 61354, "completion_tokens": 2197, "thoughts_tokens": 0, "total_tokens": 236483, "request_count": 17 } }, "agent_requests": 24, "main_agent_model_calls": 5, "tool_calls": 21, "main_agent_timed_model_calls": 5 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [182.696], "agent_gross_standard_cost_usd": 0.37392255, "judge_gross_standard_cost_usd": 0.4609935, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 792345, "agent_prompt_tokens": 773009, "agent_cached_input_tokens": 377339, "agent_thinking_tokens": 15200, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 150164, "cached_input_tokens": 0, "completion_tokens": 1122, "thoughts_tokens": 2648, "total_tokens": 153934, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 343262, "cached_input_tokens": 252441, "completion_tokens": 816, "thoughts_tokens": 12552, "total_tokens": 356630, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 279583, "cached_input_tokens": 124898, "completion_tokens": 2198, "thoughts_tokens": 0, "total_tokens": 281781, "request_count": 19 } }, "agent_requests": 29, "main_agent_model_calls": 8, "tool_calls": 27, "main_agent_timed_model_calls": 8 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [71.294], "agent_gross_standard_cost_usd": 0.222872125, "judge_gross_standard_cost_usd": 0.271242, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 403043, "agent_prompt_tokens": 397987, "agent_cached_input_tokens": 91157, "agent_thinking_tokens": 2446, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 120534, "cached_input_tokens": 0, "completion_tokens": 1105, "thoughts_tokens": 2446, "total_tokens": 124085, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 131471, "cached_input_tokens": 35286, "completion_tokens": 347, "thoughts_tokens": 0, "total_tokens": 131818, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 145982, "cached_input_tokens": 55871, "completion_tokens": 1158, "thoughts_tokens": 0, "total_tokens": 147140, "request_count": 10 } }, "agent_requests": 17, "main_agent_model_calls": 5, "tool_calls": 13, "main_agent_timed_model_calls": 5 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [76.892], "agent_gross_standard_cost_usd": 0.128595975, "judge_gross_standard_cost_usd": 0.139953, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 293355, "agent_prompt_tokens": 287343, "agent_cached_input_tokens": 91882, "agent_thinking_tokens": 3920, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 97226, "cached_input_tokens": 0, "completion_tokens": 988, "thoughts_tokens": 2608, "total_tokens": 100822, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 66390, "cached_input_tokens": 29057, "completion_tokens": 298, "thoughts_tokens": 1312, "total_tokens": 68000, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 123727, "cached_input_tokens": 62825, "completion_tokens": 806, "thoughts_tokens": 0, "total_tokens": 124533, "request_count": 8 } }, "agent_requests": 13, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.062], "agent_gross_standard_cost_usd": 0.192385575, "judge_gross_standard_cost_usd": 0.23745, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 361997, "agent_prompt_tokens": 355282, "agent_cached_input_tokens": 85555, "agent_thinking_tokens": 3988, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 115949, "cached_input_tokens": 0, "completion_tokens": 1167, "thoughts_tokens": 2691, "total_tokens": 119807, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 109364, "cached_input_tokens": 41196, "completion_tokens": 400, "thoughts_tokens": 1297, "total_tokens": 111061, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 129969, "cached_input_tokens": 44359, "completion_tokens": 1160, "thoughts_tokens": 0, "total_tokens": 131129, "request_count": 9 } }, "agent_requests": 15, "main_agent_model_calls": 4, "tool_calls": 13, "main_agent_timed_model_calls": 4 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [224.889], "agent_gross_standard_cost_usd": 0.41738156, "judge_gross_standard_cost_usd": 0.468747, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 711378, "agent_prompt_tokens": 698111, "agent_cached_input_tokens": 139058, "agent_thinking_tokens": 8382, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 147237, "cached_input_tokens": 10147, "completion_tokens": 1161, "thoughts_tokens": 2811, "total_tokens": 151209, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 230151, "cached_input_tokens": 82613, "completion_tokens": 911, "thoughts_tokens": 5571, "total_tokens": 236633, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 320723, "cached_input_tokens": 46298, "completion_tokens": 2813, "thoughts_tokens": 0, "total_tokens": 323536, "request_count": 22 } }, "agent_requests": 30, "main_agent_model_calls": 6, "tool_calls": 23, "main_agent_timed_model_calls": 6 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [275.418], "agent_gross_standard_cost_usd": 2.29752215, "judge_gross_standard_cost_usd": 1.4685585, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 2567576, "agent_prompt_tokens": 2555803, "agent_cached_input_tokens": 1335434, "agent_thinking_tokens": 3496, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 349212, "cached_input_tokens": 0, "completion_tokens": 1284, "thoughts_tokens": 3496, "total_tokens": 353992, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 1434528, "cached_input_tokens": 975114, "completion_tokens": 1389, "thoughts_tokens": 0, "total_tokens": 1435917, "request_count": 13 }, "gemini-3.1-flash-lite": { "prompt_tokens": 772063, "cached_input_tokens": 360320, "completion_tokens": 5604, "thoughts_tokens": 0, "total_tokens": 777667, "request_count": 43 } }, "agent_requests": 58, "main_agent_model_calls": 13, "tool_calls": 56, "main_agent_timed_model_calls": 13 }, { "task": "task-11", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [100.135], "agent_gross_standard_cost_usd": 0.693098575, "judge_gross_standard_cost_usd": 0.1672845, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 358725, "agent_prompt_tokens": 348650, "agent_cached_input_tokens": 58658, "agent_thinking_tokens": 7632, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 103985, "cached_input_tokens": 0, "completion_tokens": 1112, "thoughts_tokens": 2317, "total_tokens": 107414, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 169775, "cached_input_tokens": 42265, "completion_tokens": 622, "thoughts_tokens": 5315, "total_tokens": 175712, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 74890, "cached_input_tokens": 16393, "completion_tokens": 709, "thoughts_tokens": 0, "total_tokens": 75599, "request_count": 5 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 13, "main_agent_timed_model_calls": 5 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [75.871], "agent_gross_standard_cost_usd": 0.23146685, "judge_gross_standard_cost_usd": 0.2677185, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 435164, "agent_prompt_tokens": 423251, "agent_cached_input_tokens": 107659, "agent_thinking_tokens": 8275, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 121250, "cached_input_tokens": 0, "completion_tokens": 1641, "thoughts_tokens": 3783, "total_tokens": 126674, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 132494, "cached_input_tokens": 0, "completion_tokens": 1530, "thoughts_tokens": 0, "total_tokens": 134024, "request_count": 10 }, "gemini-3.8-flash": { "prompt_tokens": 169507, "cached_input_tokens": 107659, "completion_tokens": 467, "thoughts_tokens": 4492, "total_tokens": 174466, "request_count": 4 } }, "agent_requests": 16, "main_agent_model_calls": 4, "tool_calls": 12, "main_agent_timed_model_calls": 4 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [118.021], "agent_gross_standard_cost_usd": 0.280546375, "judge_gross_standard_cost_usd": 0.263778, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 457519, "agent_prompt_tokens": 448798, "agent_cached_input_tokens": 23469, "agent_thinking_tokens": 5218, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 217908, "cached_input_tokens": 0, "completion_tokens": 1451, "thoughts_tokens": 5218, "total_tokens": 224577, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 109846, "cached_input_tokens": 12849, "completion_tokens": 1448, "thoughts_tokens": 0, "total_tokens": 111294, "request_count": 9 }, "gemini-3.7-flash": { "prompt_tokens": 121044, "cached_input_tokens": 10620, "completion_tokens": 604, "thoughts_tokens": 0, "total_tokens": 121648, "request_count": 3 } }, "agent_requests": 15, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [55.327], "agent_gross_standard_cost_usd": 0.1708158, "judge_gross_standard_cost_usd": 0.2606955, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 332658, "agent_prompt_tokens": 325168, "agent_cached_input_tokens": 27333, "agent_thinking_tokens": 4173, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 116217, "cached_input_tokens": 0, "completion_tokens": 1237, "thoughts_tokens": 2931, "total_tokens": 120385, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 147347, "cached_input_tokens": 16708, "completion_tokens": 1754, "thoughts_tokens": 0, "total_tokens": 149101, "request_count": 10 }, "gemini-3.7-flash": { "prompt_tokens": 61604, "cached_input_tokens": 10625, "completion_tokens": 326, "thoughts_tokens": 1242, "total_tokens": 63172, "request_count": 2 } }, "agent_requests": 14, "main_agent_model_calls": 2, "tool_calls": 12, "main_agent_timed_model_calls": 2 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [73.797], "agent_gross_standard_cost_usd": 0.167757475, "judge_gross_standard_cost_usd": 0.2500965, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 322562, "agent_prompt_tokens": 315795, "agent_cached_input_tokens": 78702, "agent_thinking_tokens": 3963, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 110161, "cached_input_tokens": 0, "completion_tokens": 1046, "thoughts_tokens": 2892, "total_tokens": 114099, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 132485, "cached_input_tokens": 65841, "completion_tokens": 1502, "thoughts_tokens": 0, "total_tokens": 133987, "request_count": 10 }, "gemini-3.5-flash": { "prompt_tokens": 73149, "cached_input_tokens": 12861, "completion_tokens": 256, "thoughts_tokens": 1071, "total_tokens": 74476, "request_count": 3 } }, "agent_requests": 15, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [66.896], "agent_gross_standard_cost_usd": 0.22710999999999998, "judge_gross_standard_cost_usd": 0.256497, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 351497, "agent_prompt_tokens": 341335, "agent_cached_input_tokens": 41770, "agent_thinking_tokens": 7149, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 112195, "cached_input_tokens": 0, "completion_tokens": 1033, "thoughts_tokens": 3751, "total_tokens": 116979, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 139280, "cached_input_tokens": 28408, "completion_tokens": 1612, "thoughts_tokens": 0, "total_tokens": 140892, "request_count": 9 }, "gemini-3.5-flash": { "prompt_tokens": 89860, "cached_input_tokens": 13362, "completion_tokens": 368, "thoughts_tokens": 3398, "total_tokens": 93626, "request_count": 3 } }, "agent_requests": 14, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [50.847], "agent_gross_standard_cost_usd": 0.32003292499999997, "judge_gross_standard_cost_usd": 0.2555025, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 384651, "agent_prompt_tokens": 377627, "agent_cached_input_tokens": 85020, "agent_thinking_tokens": 3852, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 113551, "cached_input_tokens": 0, "completion_tokens": 1230, "thoughts_tokens": 3852, "total_tokens": 118633, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 144817, "cached_input_tokens": 19485, "completion_tokens": 1373, "thoughts_tokens": 0, "total_tokens": 146190, "request_count": 10 }, "gemini-4-argon": { "prompt_tokens": 119259, "cached_input_tokens": 65535, "completion_tokens": 569, "thoughts_tokens": 0, "total_tokens": 119828, "request_count": 3 } }, "agent_requests": 15, "main_agent_model_calls": 3, "tool_calls": 14, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [83.932], "agent_gross_standard_cost_usd": 0.505852875, "judge_gross_standard_cost_usd": 0.272634, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 402167, "agent_prompt_tokens": 389417, "agent_cached_input_tokens": 138990, "agent_thinking_tokens": 8681, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 122558, "cached_input_tokens": 0, "completion_tokens": 1773, "thoughts_tokens": 3522, "total_tokens": 127853, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 132494, "cached_input_tokens": 82093, "completion_tokens": 1576, "thoughts_tokens": 0, "total_tokens": 134070, "request_count": 10 }, "gemini-4-argon": { "prompt_tokens": 134365, "cached_input_tokens": 56897, "completion_tokens": 720, "thoughts_tokens": 5159, "total_tokens": 140244, "request_count": 3 } }, "agent_requests": 15, "main_agent_model_calls": 3, "tool_calls": 13, "main_agent_timed_model_calls": 3 }, { "task": "task-12", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "shopping/cross-sell", "support/escalation", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [56.199], "agent_gross_standard_cost_usd": 0.1901451, "judge_gross_standard_cost_usd": 0.2667195, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 529602, "agent_prompt_tokens": 522669, "agent_cached_input_tokens": 280726, "agent_thinking_tokens": 3900, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 121249, "cached_input_tokens": 0, "completion_tokens": 1108, "thoughts_tokens": 3900, "total_tokens": 126257, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 132485, "cached_input_tokens": 69242, "completion_tokens": 1438, "thoughts_tokens": 0, "total_tokens": 133923, "request_count": 10 }, "gemini-3.8-flash": { "prompt_tokens": 268935, "cached_input_tokens": 211484, "completion_tokens": 487, "thoughts_tokens": 0, "total_tokens": 269422, "request_count": 6 } }, "agent_requests": 18, "main_agent_model_calls": 6, "tool_calls": 12, "main_agent_timed_model_calls": 6 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [54.673], "agent_gross_standard_cost_usd": 0.14776975, "judge_gross_standard_cost_usd": 0.2254425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 333625, "agent_prompt_tokens": 327084, "agent_cached_input_tokens": 99965, "agent_thinking_tokens": 4201, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 113725, "cached_input_tokens": 0, "completion_tokens": 1030, "thoughts_tokens": 2805, "total_tokens": 117560, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 86684, "cached_input_tokens": 44191, "completion_tokens": 357, "thoughts_tokens": 1396, "total_tokens": 88437, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 126675, "cached_input_tokens": 55774, "completion_tokens": 953, "thoughts_tokens": 0, "total_tokens": 127628, "request_count": 9 } }, "agent_requests": 14, "main_agent_model_calls": 3, "tool_calls": 12, "main_agent_timed_model_calls": 3 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [69.277], "agent_gross_standard_cost_usd": 0.189709025, "judge_gross_standard_cost_usd": 0.341838, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 434294, "agent_prompt_tokens": 426815, "agent_cached_input_tokens": 137951, "agent_thinking_tokens": 4431, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 131805, "cached_input_tokens": 0, "completion_tokens": 1115, "thoughts_tokens": 2831, "total_tokens": 135751, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 140860, "cached_input_tokens": 85772, "completion_tokens": 518, "thoughts_tokens": 1600, "total_tokens": 142978, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 154150, "cached_input_tokens": 52179, "completion_tokens": 1415, "thoughts_tokens": 0, "total_tokens": 155565, "request_count": 11 } }, "agent_requests": 17, "main_agent_model_calls": 4, "tool_calls": 17, "main_agent_timed_model_calls": 4 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [99.344], "agent_gross_standard_cost_usd": 0.323196725, "judge_gross_standard_cost_usd": 0.2339895, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 581353, "agent_prompt_tokens": 571172, "agent_cached_input_tokens": 148003, "agent_thinking_tokens": 6759, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 214358, "cached_input_tokens": 0, "completion_tokens": 1640, "thoughts_tokens": 5216, "total_tokens": 221214, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 214343, "cached_input_tokens": 92122, "completion_tokens": 666, "thoughts_tokens": 1543, "total_tokens": 216552, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 142471, "cached_input_tokens": 55881, "completion_tokens": 1116, "thoughts_tokens": 0, "total_tokens": 143587, "request_count": 10 } }, "agent_requests": 19, "main_agent_model_calls": 6, "tool_calls": 14, "main_agent_timed_model_calls": 6 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [142.939], "agent_gross_standard_cost_usd": 0.294867875, "judge_gross_standard_cost_usd": 0.343254, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 549069, "agent_prompt_tokens": 536701, "agent_cached_input_tokens": 198456, "agent_thinking_tokens": 8940, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 136587, "cached_input_tokens": 0, "completion_tokens": 1182, "thoughts_tokens": 3280, "total_tokens": 141049, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 218481, "cached_input_tokens": 127555, "completion_tokens": 460, "thoughts_tokens": 5660, "total_tokens": 224601, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 181633, "cached_input_tokens": 70901, "completion_tokens": 1786, "thoughts_tokens": 0, "total_tokens": 183419, "request_count": 16 } }, "agent_requests": 24, "main_agent_model_calls": 6, "tool_calls": 23, "main_agent_timed_model_calls": 6 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [356.416], "agent_gross_standard_cost_usd": 2.41861095, "judge_gross_standard_cost_usd": 1.337739, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 4210055, "agent_prompt_tokens": 4195468, "agent_cached_input_tokens": 3027298, "agent_thinking_tokens": 5511, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 285735, "cached_input_tokens": 0, "completion_tokens": 1052, "thoughts_tokens": 3410, "total_tokens": 290197, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 3207759, "cached_input_tokens": 2813200, "completion_tokens": 2073, "thoughts_tokens": 0, "total_tokens": 3209832, "request_count": 30 }, "gemini-3.1-flash-lite": { "prompt_tokens": 701974, "cached_input_tokens": 214098, "completion_tokens": 5951, "thoughts_tokens": 2101, "total_tokens": 710026, "request_count": 46 } }, "agent_requests": 78, "main_agent_model_calls": 30, "tool_calls": 76, "main_agent_timed_model_calls": 30 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [72.858], "agent_gross_standard_cost_usd": 0.4529324, "judge_gross_standard_cost_usd": 0.136164, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 265401, "agent_prompt_tokens": 255066, "agent_cached_input_tokens": 54786, "agent_thinking_tokens": 8239, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 98919, "cached_input_tokens": 0, "completion_tokens": 1212, "thoughts_tokens": 3512, "total_tokens": 103643, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 114047, "cached_input_tokens": 41766, "completion_tokens": 568, "thoughts_tokens": 4727, "total_tokens": 119342, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 42100, "cached_input_tokens": 13020, "completion_tokens": 316, "thoughts_tokens": 0, "total_tokens": 42416, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 10, "main_agent_timed_model_calls": 4 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [77.033], "agent_gross_standard_cost_usd": 0.21744485, "judge_gross_standard_cost_usd": 0.32145, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 467861, "agent_prompt_tokens": 460244, "agent_cached_input_tokens": 112369, "agent_thinking_tokens": 4693, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 123700, "cached_input_tokens": 0, "completion_tokens": 1100, "thoughts_tokens": 2656, "total_tokens": 127456, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 135926, "cached_input_tokens": 69809, "completion_tokens": 451, "thoughts_tokens": 2037, "total_tokens": 138414, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 200618, "cached_input_tokens": 42560, "completion_tokens": 1373, "thoughts_tokens": 0, "total_tokens": 201991, "request_count": 14 } }, "agent_requests": 20, "main_agent_model_calls": 4, "tool_calls": 19, "main_agent_timed_model_calls": 4 }, { "task": "task-13", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [135.068], "agent_gross_standard_cost_usd": 0.3019906, "judge_gross_standard_cost_usd": 0.3681945, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 544107, "agent_prompt_tokens": 529313, "agent_cached_input_tokens": 180475, "agent_thinking_tokens": 11559, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 132227, "cached_input_tokens": 0, "completion_tokens": 1107, "thoughts_tokens": 2530, "total_tokens": 135864, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 220230, "cached_input_tokens": 137701, "completion_tokens": 547, "thoughts_tokens": 9029, "total_tokens": 229806, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 176856, "cached_input_tokens": 42774, "completion_tokens": 1581, "thoughts_tokens": 0, "total_tokens": 178437, "request_count": 13 } }, "agent_requests": 21, "main_agent_model_calls": 6, "tool_calls": 22, "main_agent_timed_model_calls": 6 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.227], "agent_gross_standard_cost_usd": 0.07476555, "judge_gross_standard_cost_usd": 0.024753, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 101269, "agent_prompt_tokens": 96550, "agent_cached_input_tokens": 16305, "agent_thinking_tokens": 3616, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56346, "cached_input_tokens": 0, "completion_tokens": 926, "thoughts_tokens": 2239, "total_tokens": 59511, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 40204, "cached_input_tokens": 16305, "completion_tokens": 177, "thoughts_tokens": 1377, "total_tokens": 41758, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 2, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [71.45], "agent_gross_standard_cost_usd": 0.090419025, "judge_gross_standard_cost_usd": 0.0403335, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 177393, "agent_prompt_tokens": 171195, "agent_cached_input_tokens": 40296, "agent_thinking_tokens": 4232, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54455, "cached_input_tokens": 0, "completion_tokens": 1049, "thoughts_tokens": 3305, "total_tokens": 58809, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 54953, "cached_input_tokens": 33165, "completion_tokens": 264, "thoughts_tokens": 927, "total_tokens": 56144, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 61787, "cached_input_tokens": 7131, "completion_tokens": 653, "thoughts_tokens": 0, "total_tokens": 62440, "request_count": 9 } }, "agent_requests": 14, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [90.132], "agent_gross_standard_cost_usd": 0.11578382499999999, "judge_gross_standard_cost_usd": 0.0923235, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 214696, "agent_prompt_tokens": 206727, "agent_cached_input_tokens": 64068, "agent_thinking_tokens": 5499, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60990, "cached_input_tokens": 0, "completion_tokens": 1028, "thoughts_tokens": 2762, "total_tokens": 64780, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 59664, "cached_input_tokens": 33125, "completion_tokens": 251, "thoughts_tokens": 2737, "total_tokens": 62652, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 86073, "cached_input_tokens": 30943, "completion_tokens": 1191, "thoughts_tokens": 0, "total_tokens": 87264, "request_count": 7 } }, "agent_requests": 12, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-14", "suite": "Product measurements", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.11], "agent_gross_standard_cost_usd": 0.2414348, "judge_gross_standard_cost_usd": 0.132177, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 247572, "agent_prompt_tokens": 243724, "agent_cached_input_tokens": 93239, "agent_thinking_tokens": 1919, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 76644, "cached_input_tokens": 0, "completion_tokens": 810, "thoughts_tokens": 1919, "total_tokens": 79373, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 117725, "cached_input_tokens": 71945, "completion_tokens": 264, "thoughts_tokens": 0, "total_tokens": 117989, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 49355, "cached_input_tokens": 21294, "completion_tokens": 855, "thoughts_tokens": 0, "total_tokens": 50210, "request_count": 4 } }, "agent_requests": 10, "main_agent_model_calls": 4, "tool_calls": 9, "main_agent_timed_model_calls": 4 }, { "task": "task-14", "suite": "Product measurements", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [38.368], "agent_gross_standard_cost_usd": 0.2026741, "judge_gross_standard_cost_usd": 0.0294195, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140661, "agent_prompt_tokens": 135626, "agent_cached_input_tokens": 47137, "agent_thinking_tokens": 3980, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57214, "cached_input_tokens": 0, "completion_tokens": 843, "thoughts_tokens": 1998, "total_tokens": 60055, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 78412, "cached_input_tokens": 47137, "completion_tokens": 212, "thoughts_tokens": 1982, "total_tokens": 80606, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-14", "suite": "Product measurements", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.988], "agent_gross_standard_cost_usd": 0.07578599999999999, "judge_gross_standard_cost_usd": 0.0244005, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 100721, "agent_prompt_tokens": 96228, "agent_cached_input_tokens": 14474, "agent_thinking_tokens": 3569, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56328, "cached_input_tokens": 0, "completion_tokens": 752, "thoughts_tokens": 2272, "total_tokens": 59352, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 39900, "cached_input_tokens": 14474, "completion_tokens": 172, "thoughts_tokens": 1297, "total_tokens": 41369, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-14", "suite": "Product measurements", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.696], "agent_gross_standard_cost_usd": 0.13949765, "judge_gross_standard_cost_usd": 0.0284175, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 168637, "agent_prompt_tokens": 158750, "agent_cached_input_tokens": 62607, "agent_thinking_tokens": 8825, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57217, "cached_input_tokens": 0, "completion_tokens": 843, "thoughts_tokens": 3077, "total_tokens": 61137, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 101533, "cached_input_tokens": 62607, "completion_tokens": 219, "thoughts_tokens": 5748, "total_tokens": 107500, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-14", "suite": "Product measurements", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.826], "agent_gross_standard_cost_usd": 0.133983375, "judge_gross_standard_cost_usd": 0.14085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 207240, "agent_prompt_tokens": 203270, "agent_cached_input_tokens": 35226, "agent_thinking_tokens": 2425, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 75387, "cached_input_tokens": 0, "completion_tokens": 826, "thoughts_tokens": 1612, "total_tokens": 77825, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 77377, "cached_input_tokens": 21239, "completion_tokens": 204, "thoughts_tokens": 813, "total_tokens": 78394, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 50506, "cached_input_tokens": 13987, "completion_tokens": 515, "thoughts_tokens": 0, "total_tokens": 51021, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.352], "agent_gross_standard_cost_usd": 0.06565585, "judge_gross_standard_cost_usd": 0.026382, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 120915, "agent_prompt_tokens": 116753, "agent_cached_input_tokens": 38231, "agent_thinking_tokens": 2796, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51544, "cached_input_tokens": 0, "completion_tokens": 927, "thoughts_tokens": 2424, "total_tokens": 54895, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12889, "cached_input_tokens": 5114, "completion_tokens": 177, "thoughts_tokens": 0, "total_tokens": 13066, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 52320, "cached_input_tokens": 33117, "completion_tokens": 262, "thoughts_tokens": 372, "total_tokens": 52954, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [124.626], "agent_gross_standard_cost_usd": 0.103902925, "judge_gross_standard_cost_usd": 0.0411405, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 154679, "agent_prompt_tokens": 147607, "agent_cached_input_tokens": 52434, "agent_thinking_tokens": 5537, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57804, "cached_input_tokens": 0, "completion_tokens": 1008, "thoughts_tokens": 2598, "total_tokens": 61410, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 23875, "cached_input_tokens": 12547, "completion_tokens": 245, "thoughts_tokens": 0, "total_tokens": 24120, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 65928, "cached_input_tokens": 39887, "completion_tokens": 282, "thoughts_tokens": 2939, "total_tokens": 69149, "request_count": 3 } }, "agent_requests": 9, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.856], "agent_gross_standard_cost_usd": 0.15020915, "judge_gross_standard_cost_usd": 0.036663, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 121205, "agent_prompt_tokens": 117472, "agent_cached_input_tokens": 23606, "agent_thinking_tokens": 2347, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57775, "cached_input_tokens": 0, "completion_tokens": 932, "thoughts_tokens": 2347, "total_tokens": 61054, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12909, "cached_input_tokens": 5116, "completion_tokens": 182, "thoughts_tokens": 0, "total_tokens": 13091, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 46788, "cached_input_tokens": 18490, "completion_tokens": 272, "thoughts_tokens": 0, "total_tokens": 47060, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [34.621], "agent_gross_standard_cost_usd": 0.210470725, "judge_gross_standard_cost_usd": 0.0515835, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 166856, "agent_prompt_tokens": 162052, "agent_cached_input_tokens": 54363, "agent_thinking_tokens": 3328, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62897, "cached_input_tokens": 0, "completion_tokens": 862, "thoughts_tokens": 2240, "total_tokens": 65999, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14535, "cached_input_tokens": 5317, "completion_tokens": 256, "thoughts_tokens": 0, "total_tokens": 14791, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 84620, "cached_input_tokens": 49046, "completion_tokens": 358, "thoughts_tokens": 1088, "total_tokens": 86066, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-15", "suite": "Product measurements", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.741], "agent_gross_standard_cost_usd": 0.063685275, "judge_gross_standard_cost_usd": 0.030372, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 113057, "agent_prompt_tokens": 109762, "agent_cached_input_tokens": 26381, "agent_thinking_tokens": 1791, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51565, "cached_input_tokens": 0, "completion_tokens": 1046, "thoughts_tokens": 1193, "total_tokens": 53804, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 23906, "cached_input_tokens": 12549, "completion_tokens": 203, "thoughts_tokens": 0, "total_tokens": 24109, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 34291, "cached_input_tokens": 13832, "completion_tokens": 255, "thoughts_tokens": 598, "total_tokens": 35144, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [44.944], "agent_gross_standard_cost_usd": 0.09615795, "judge_gross_standard_cost_usd": 0.0432285, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 128961, "agent_prompt_tokens": 122522, "agent_cached_input_tokens": 14395, "agent_thinking_tokens": 4932, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57804, "cached_input_tokens": 0, "completion_tokens": 938, "thoughts_tokens": 2196, "total_tokens": 60938, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 23902, "cached_input_tokens": 0, "completion_tokens": 262, "thoughts_tokens": 0, "total_tokens": 24164, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 40816, "cached_input_tokens": 14395, "completion_tokens": 307, "thoughts_tokens": 2736, "total_tokens": 43859, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.057], "agent_gross_standard_cost_usd": 0.06923635, "judge_gross_standard_cost_usd": 0.0399255, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116509, "agent_prompt_tokens": 112720, "agent_cached_input_tokens": 21472, "agent_thinking_tokens": 2047, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57846, "cached_input_tokens": 0, "completion_tokens": 1168, "thoughts_tokens": 2047, "total_tokens": 61061, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14525, "cached_input_tokens": 5316, "completion_tokens": 268, "thoughts_tokens": 0, "total_tokens": 14793, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 40349, "cached_input_tokens": 16156, "completion_tokens": 306, "thoughts_tokens": 0, "total_tokens": 40655, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-15", "suite": "Product measurements", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [31.13], "agent_gross_standard_cost_usd": 0.0822141, "judge_gross_standard_cost_usd": 0.039429, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 141037, "agent_prompt_tokens": 137146, "agent_cached_input_tokens": 41349, "agent_thinking_tokens": 2439, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57755, "cached_input_tokens": 0, "completion_tokens": 964, "thoughts_tokens": 1805, "total_tokens": 60524, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14523, "cached_input_tokens": 5316, "completion_tokens": 257, "thoughts_tokens": 0, "total_tokens": 14780, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 64868, "cached_input_tokens": 36033, "completion_tokens": 231, "thoughts_tokens": 634, "total_tokens": 65733, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.152], "agent_gross_standard_cost_usd": 0.0759413, "judge_gross_standard_cost_usd": 0.0270705, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122028, "agent_prompt_tokens": 117305, "agent_cached_input_tokens": 33180, "agent_thinking_tokens": 3225, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51531, "cached_input_tokens": 0, "completion_tokens": 1041, "thoughts_tokens": 1781, "total_tokens": 54353, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12906, "cached_input_tokens": 0, "completion_tokens": 189, "thoughts_tokens": 0, "total_tokens": 13095, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 52868, "cached_input_tokens": 33180, "completion_tokens": 268, "thoughts_tokens": 1444, "total_tokens": 54580, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [22.448], "agent_gross_standard_cost_usd": 0.156016625, "judge_gross_standard_cost_usd": 0.036594, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 120448, "agent_prompt_tokens": 117414, "agent_cached_input_tokens": 21499, "agent_thinking_tokens": 1652, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57739, "cached_input_tokens": 0, "completion_tokens": 945, "thoughts_tokens": 1652, "total_tokens": 60336, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12908, "cached_input_tokens": 5115, "completion_tokens": 185, "thoughts_tokens": 0, "total_tokens": 13093, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 46767, "cached_input_tokens": 16384, "completion_tokens": 252, "thoughts_tokens": 0, "total_tokens": 47019, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-16", "suite": "Product measurements", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.638], "agent_gross_standard_cost_usd": 0.18989530000000002, "judge_gross_standard_cost_usd": 0.0457635, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 158563, "agent_prompt_tokens": 154303, "agent_cached_input_tokens": 53299, "agent_thinking_tokens": 2844, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59191, "cached_input_tokens": 0, "completion_tokens": 855, "thoughts_tokens": 2051, "total_tokens": 62097, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14528, "cached_input_tokens": 5316, "completion_tokens": 251, "thoughts_tokens": 0, "total_tokens": 14779, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 80584, "cached_input_tokens": 47983, "completion_tokens": 310, "thoughts_tokens": 793, "total_tokens": 81687, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 10, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [20.784], "agent_gross_standard_cost_usd": 0.058736625, "judge_gross_standard_cost_usd": 0.027588, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 101580, "agent_prompt_tokens": 98537, "agent_cached_input_tokens": 18933, "agent_thinking_tokens": 1700, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51506, "cached_input_tokens": 0, "completion_tokens": 915, "thoughts_tokens": 1700, "total_tokens": 54121, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 12904, "cached_input_tokens": 5115, "completion_tokens": 179, "thoughts_tokens": 0, "total_tokens": 13083, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 34127, "cached_input_tokens": 13818, "completion_tokens": 249, "thoughts_tokens": 0, "total_tokens": 34376, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-16", "suite": "Product measurements", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.22], "agent_gross_standard_cost_usd": 0.1029718, "judge_gross_standard_cost_usd": 0.04092, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 146873, "agent_prompt_tokens": 140560, "agent_cached_input_tokens": 41414, "agent_thinking_tokens": 4810, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58349, "cached_input_tokens": 0, "completion_tokens": 1021, "thoughts_tokens": 2155, "total_tokens": 61525, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14522, "cached_input_tokens": 5316, "completion_tokens": 206, "thoughts_tokens": 0, "total_tokens": 14728, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 67689, "cached_input_tokens": 36098, "completion_tokens": 276, "thoughts_tokens": 2655, "total_tokens": 70620, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.512], "agent_gross_standard_cost_usd": 0.0741411, "judge_gross_standard_cost_usd": 0.041436, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116408, "agent_prompt_tokens": 112681, "agent_cached_input_tokens": 14366, "agent_thinking_tokens": 2210, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57769, "cached_input_tokens": 0, "completion_tokens": 1059, "thoughts_tokens": 1886, "total_tokens": 60714, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14524, "cached_input_tokens": 0, "completion_tokens": 206, "thoughts_tokens": 0, "total_tokens": 14730, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 40388, "cached_input_tokens": 14366, "completion_tokens": 252, "thoughts_tokens": 324, "total_tokens": 40964, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.097], "agent_gross_standard_cost_usd": 0.072678925, "judge_gross_standard_cost_usd": 0.0401235, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 115951, "agent_prompt_tokens": 112608, "agent_cached_input_tokens": 19685, "agent_thinking_tokens": 1861, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57711, "cached_input_tokens": 0, "completion_tokens": 1002, "thoughts_tokens": 1383, "total_tokens": 60096, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14512, "cached_input_tokens": 5315, "completion_tokens": 232, "thoughts_tokens": 0, "total_tokens": 14744, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 40385, "cached_input_tokens": 14370, "completion_tokens": 248, "thoughts_tokens": 478, "total_tokens": 41111, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 7, "main_agent_timed_model_calls": 2 }, { "task": "task-16", "suite": "Product measurements", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.355], "agent_gross_standard_cost_usd": 0.09963695, "judge_gross_standard_cost_usd": 0.040113, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140810, "agent_prompt_tokens": 136899, "agent_cached_input_tokens": 28743, "agent_thinking_tokens": 2308, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57795, "cached_input_tokens": 0, "completion_tokens": 1097, "thoughts_tokens": 1839, "total_tokens": 60731, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 14520, "cached_input_tokens": 5316, "completion_tokens": 200, "thoughts_tokens": 0, "total_tokens": 14720, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 64584, "cached_input_tokens": 23427, "completion_tokens": 306, "thoughts_tokens": 469, "total_tokens": 65359, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 7, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.511], "agent_gross_standard_cost_usd": 0.162229075, "judge_gross_standard_cost_usd": 0.039756, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 138334, "agent_prompt_tokens": 132855, "agent_cached_input_tokens": 23510, "agent_thinking_tokens": 3543, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57005, "cached_input_tokens": 0, "completion_tokens": 1209, "thoughts_tokens": 3543, "total_tokens": 61757, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 28452, "cached_input_tokens": 5061, "completion_tokens": 365, "thoughts_tokens": 0, "total_tokens": 28817, "request_count": 7 }, "gemini-4-argon": { "prompt_tokens": 47398, "cached_input_tokens": 18449, "completion_tokens": 362, "thoughts_tokens": 0, "total_tokens": 47760, "request_count": 2 } }, "agent_requests": 11, "main_agent_model_calls": 2, "tool_calls": 10, "main_agent_timed_model_calls": 2 }, { "task": "task-17", "suite": "Product measurements", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [142.95], "agent_gross_standard_cost_usd": 0.5005843, "judge_gross_standard_cost_usd": 0.04761, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 185186, "agent_prompt_tokens": 165751, "agent_cached_input_tokens": 50592, "agent_thinking_tokens": 17609, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61848, "cached_input_tokens": 0, "completion_tokens": 1016, "thoughts_tokens": 2328, "total_tokens": 65192, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17072, "cached_input_tokens": 0, "completion_tokens": 405, "thoughts_tokens": 0, "total_tokens": 17477, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 86831, "cached_input_tokens": 50592, "completion_tokens": 405, "thoughts_tokens": 15281, "total_tokens": 102517, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 11, "main_agent_timed_model_calls": 2 }, { "task": "task-17", "suite": "Product measurements", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.414], "agent_gross_standard_cost_usd": 0.07997504999999999, "judge_gross_standard_cost_usd": 0.0389715, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155488, "agent_prompt_tokens": 150956, "agent_cached_input_tokens": 56612, "agent_thinking_tokens": 2937, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 54890, "cached_input_tokens": 0, "completion_tokens": 964, "thoughts_tokens": 2937, "total_tokens": 58791, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16388, "cached_input_tokens": 3402, "completion_tokens": 332, "thoughts_tokens": 0, "total_tokens": 16720, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 79678, "cached_input_tokens": 53210, "completion_tokens": 299, "thoughts_tokens": 0, "total_tokens": 79977, "request_count": 4 } }, "agent_requests": 10, "main_agent_model_calls": 4, "tool_calls": 8, "main_agent_timed_model_calls": 4 }, { "task": "task-17", "suite": "Product measurements", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [117.048], "agent_gross_standard_cost_usd": 0.223260075, "judge_gross_standard_cost_usd": 0.056127, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 285714, "agent_prompt_tokens": 268207, "agent_cached_input_tokens": 136363, "agent_thinking_tokens": 15460, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62890, "cached_input_tokens": 0, "completion_tokens": 1122, "thoughts_tokens": 2796, "total_tokens": 66808, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32229, "cached_input_tokens": 10707, "completion_tokens": 494, "thoughts_tokens": 0, "total_tokens": 32723, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 173088, "cached_input_tokens": 125656, "completion_tokens": 431, "thoughts_tokens": 12664, "total_tokens": 186183, "request_count": 6 } }, "agent_requests": 15, "main_agent_model_calls": 6, "tool_calls": 16, "main_agent_timed_model_calls": 6 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [46.876], "agent_gross_standard_cost_usd": 0.097307125, "judge_gross_standard_cost_usd": 0.038361, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155841, "agent_prompt_tokens": 148765, "agent_cached_input_tokens": 46478, "agent_thinking_tokens": 5457, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56801, "cached_input_tokens": 0, "completion_tokens": 1041, "thoughts_tokens": 3594, "total_tokens": 61436, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 27754, "cached_input_tokens": 10389, "completion_tokens": 304, "thoughts_tokens": 0, "total_tokens": 28058, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 64210, "cached_input_tokens": 36089, "completion_tokens": 274, "thoughts_tokens": 1863, "total_tokens": 66347, "request_count": 3 } }, "agent_requests": 11, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.998], "agent_gross_standard_cost_usd": 0.072657425, "judge_gross_standard_cost_usd": 0.028515, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 106198, "agent_prompt_tokens": 100561, "agent_cached_input_tokens": 19235, "agent_thinking_tokens": 4062, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50461, "cached_input_tokens": 0, "completion_tokens": 1039, "thoughts_tokens": 2522, "total_tokens": 54022, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16418, "cached_input_tokens": 5355, "completion_tokens": 273, "thoughts_tokens": 0, "total_tokens": 16691, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 33682, "cached_input_tokens": 13880, "completion_tokens": 263, "thoughts_tokens": 1540, "total_tokens": 35485, "request_count": 2 } }, "agent_requests": 8, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.218], "agent_gross_standard_cost_usd": 0.11927544999999999, "judge_gross_standard_cost_usd": 0.039777, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 143130, "agent_prompt_tokens": 138046, "agent_cached_input_tokens": 14511, "agent_thinking_tokens": 3355, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56756, "cached_input_tokens": 0, "completion_tokens": 1082, "thoughts_tokens": 2293, "total_tokens": 60131, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17050, "cached_input_tokens": 0, "completion_tokens": 347, "thoughts_tokens": 0, "total_tokens": 17397, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 64240, "cached_input_tokens": 14511, "completion_tokens": 300, "thoughts_tokens": 1062, "total_tokens": 65602, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-17", "suite": "Product measurements", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "hallucination/policy", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [55.75], "agent_gross_standard_cost_usd": 0.1022169, "judge_gross_standard_cost_usd": 0.036939, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 145564, "agent_prompt_tokens": 138498, "agent_cached_input_tokens": 43451, "agent_thinking_tokens": 5500, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56680, "cached_input_tokens": 0, "completion_tokens": 996, "thoughts_tokens": 2794, "total_tokens": 60470, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17054, "cached_input_tokens": 5354, "completion_tokens": 336, "thoughts_tokens": 0, "total_tokens": 17390, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 64764, "cached_input_tokens": 38097, "completion_tokens": 234, "thoughts_tokens": 2706, "total_tokens": 67704, "request_count": 3 } }, "agent_requests": 10, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [49.782], "agent_gross_standard_cost_usd": 0.3592626, "judge_gross_standard_cost_usd": 0.032922, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 174782, "agent_prompt_tokens": 167606, "agent_cached_input_tokens": 45056, "agent_thinking_tokens": 5917, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59553, "cached_input_tokens": 0, "completion_tokens": 917, "thoughts_tokens": 2690, "total_tokens": 63160, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 108053, "cached_input_tokens": 45056, "completion_tokens": 342, "thoughts_tokens": 3227, "total_tokens": 111622, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [33.65], "agent_gross_standard_cost_usd": 0.08279229999999999, "judge_gross_standard_cost_usd": 0.025395, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 104974, "agent_prompt_tokens": 99060, "agent_cached_input_tokens": 14494, "agent_thinking_tokens": 4772, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58984, "cached_input_tokens": 0, "completion_tokens": 953, "thoughts_tokens": 3008, "total_tokens": 62945, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 40076, "cached_input_tokens": 14494, "completion_tokens": 189, "thoughts_tokens": 1764, "total_tokens": 42029, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [84.183], "agent_gross_standard_cost_usd": 0.18523735, "judge_gross_standard_cost_usd": 0.038514, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 200804, "agent_prompt_tokens": 187561, "agent_cached_input_tokens": 78983, "agent_thinking_tokens": 12152, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63683, "cached_input_tokens": 0, "completion_tokens": 781, "thoughts_tokens": 1695, "total_tokens": 66159, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 123878, "cached_input_tokens": 78983, "completion_tokens": 310, "thoughts_tokens": 10457, "total_tokens": 134645, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.081], "agent_gross_standard_cost_usd": 0.0750149, "judge_gross_standard_cost_usd": 0.0267465, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 103737, "agent_prompt_tokens": 99562, "agent_cached_input_tokens": 14548, "agent_thinking_tokens": 3128, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58659, "cached_input_tokens": 0, "completion_tokens": 844, "thoughts_tokens": 2278, "total_tokens": 61781, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 40903, "cached_input_tokens": 14548, "completion_tokens": 203, "thoughts_tokens": 850, "total_tokens": 41956, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.609], "agent_gross_standard_cost_usd": 0.07664524, "judge_gross_standard_cost_usd": 0.0306105, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 106462, "agent_prompt_tokens": 102120, "agent_cached_input_tokens": 28880, "agent_thinking_tokens": 3272, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58847, "cached_input_tokens": 14208, "completion_tokens": 841, "thoughts_tokens": 2127, "total_tokens": 61815, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 43273, "cached_input_tokens": 14672, "completion_tokens": 229, "thoughts_tokens": 1145, "total_tokens": 44647, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [62.927], "agent_gross_standard_cost_usd": 0.07822595, "judge_gross_standard_cost_usd": 0.031551, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 105917, "agent_prompt_tokens": 100746, "agent_cached_input_tokens": 16501, "agent_thinking_tokens": 3975, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58706, "cached_input_tokens": 0, "completion_tokens": 938, "thoughts_tokens": 3171, "total_tokens": 62815, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 42040, "cached_input_tokens": 16501, "completion_tokens": 258, "thoughts_tokens": 804, "total_tokens": 43102, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.477], "agent_gross_standard_cost_usd": 0.09889645, "judge_gross_standard_cost_usd": 0.0317565, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109372, "agent_prompt_tokens": 102505, "agent_cached_input_tokens": 16581, "agent_thinking_tokens": 5707, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58866, "cached_input_tokens": 0, "completion_tokens": 897, "thoughts_tokens": 2740, "total_tokens": 62503, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 43639, "cached_input_tokens": 16581, "completion_tokens": 263, "thoughts_tokens": 2967, "total_tokens": 46869, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-18", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.693], "agent_gross_standard_cost_usd": 0.17589559999999999, "judge_gross_standard_cost_usd": 0.0319365, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 144359, "agent_prompt_tokens": 140958, "agent_cached_input_tokens": 47914, "agent_thinking_tokens": 2162, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59436, "cached_input_tokens": 0, "completion_tokens": 922, "thoughts_tokens": 2162, "total_tokens": 62520, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 81522, "cached_input_tokens": 47914, "completion_tokens": 317, "thoughts_tokens": 0, "total_tokens": 81839, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [22.888], "agent_gross_standard_cost_usd": 0.06825115000000001, "judge_gross_standard_cost_usd": 0.0286035, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 102401, "agent_prompt_tokens": 98402, "agent_cached_input_tokens": 14075, "agent_thinking_tokens": 2659, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59248, "cached_input_tokens": 0, "completion_tokens": 956, "thoughts_tokens": 2659, "total_tokens": 62863, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 38496, "cached_input_tokens": 14075, "completion_tokens": 326, "thoughts_tokens": 0, "total_tokens": 38822, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [102.073], "agent_gross_standard_cost_usd": 0.1980208, "judge_gross_standard_cost_usd": 0.040665, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 215885, "agent_prompt_tokens": 198832, "agent_cached_input_tokens": 94576, "agent_thinking_tokens": 15196, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 64593, "cached_input_tokens": 0, "completion_tokens": 1145, "thoughts_tokens": 3214, "total_tokens": 68952, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 133581, "cached_input_tokens": 94576, "completion_tokens": 659, "thoughts_tokens": 11982, "total_tokens": 146222, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 53, "thoughts_tokens": 0, "total_tokens": 711, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 7, "main_agent_timed_model_calls": 5 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [19.809], "agent_gross_standard_cost_usd": 0.06505599999999999, "judge_gross_standard_cost_usd": 0.0289275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 102104, "agent_prompt_tokens": 98442, "agent_cached_input_tokens": 15834, "agent_thinking_tokens": 2164, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59263, "cached_input_tokens": 0, "completion_tokens": 1118, "thoughts_tokens": 2164, "total_tokens": 62545, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 38521, "cached_input_tokens": 15834, "completion_tokens": 322, "thoughts_tokens": 0, "total_tokens": 38843, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [25.241], "agent_gross_standard_cost_usd": 0.0811515, "judge_gross_standard_cost_usd": 0.0386385, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 124731, "agent_prompt_tokens": 120226, "agent_cached_input_tokens": 31694, "agent_thinking_tokens": 3070, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59248, "cached_input_tokens": 0, "completion_tokens": 1076, "thoughts_tokens": 2508, "total_tokens": 62832, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 60320, "cached_input_tokens": 31694, "completion_tokens": 301, "thoughts_tokens": 562, "total_tokens": 61183, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [104.032], "agent_gross_standard_cost_usd": 0.0803792, "judge_gross_standard_cost_usd": 0.033942, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122713, "agent_prompt_tokens": 117357, "agent_cached_input_tokens": 33258, "agent_thinking_tokens": 3841, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59250, "cached_input_tokens": 0, "completion_tokens": 1216, "thoughts_tokens": 2877, "total_tokens": 63343, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 57449, "cached_input_tokens": 33258, "completion_tokens": 241, "thoughts_tokens": 964, "total_tokens": 58654, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [52.774], "agent_gross_standard_cost_usd": 0.1372807, "judge_gross_standard_cost_usd": 0.0384495, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 205118, "agent_prompt_tokens": 195667, "agent_cached_input_tokens": 103056, "agent_thinking_tokens": 7805, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60551, "cached_input_tokens": 0, "completion_tokens": 990, "thoughts_tokens": 3128, "total_tokens": 64669, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1532, "cached_input_tokens": 0, "completion_tokens": 108, "thoughts_tokens": 0, "total_tokens": 1640, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 133584, "cached_input_tokens": 103056, "completion_tokens": 548, "thoughts_tokens": 4677, "total_tokens": 138809, "request_count": 6 } }, "agent_requests": 10, "main_agent_model_calls": 6, "tool_calls": 6, "main_agent_timed_model_calls": 6 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [23.4], "agent_gross_standard_cost_usd": 0.139969, "judge_gross_standard_cost_usd": 0.0281805, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109702, "agent_prompt_tokens": 105831, "agent_cached_input_tokens": 20222, "agent_thinking_tokens": 2369, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59582, "cached_input_tokens": 0, "completion_tokens": 1056, "thoughts_tokens": 2369, "total_tokens": 63007, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 45591, "cached_input_tokens": 20222, "completion_tokens": 388, "thoughts_tokens": 0, "total_tokens": 45979, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-19", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [45.468], "agent_gross_standard_cost_usd": 0.21552179999999999, "judge_gross_standard_cost_usd": 0.041661, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 146811, "agent_prompt_tokens": 140856, "agent_cached_input_tokens": 45768, "agent_thinking_tokens": 4352, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63834, "cached_input_tokens": 0, "completion_tokens": 1102, "thoughts_tokens": 1947, "total_tokens": 66883, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 76364, "cached_input_tokens": 45768, "completion_tokens": 443, "thoughts_tokens": 2405, "total_tokens": 79212, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [61.673], "agent_gross_standard_cost_usd": 0.1209751, "judge_gross_standard_cost_usd": 0.029739, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 107205, "agent_prompt_tokens": 95981, "agent_cached_input_tokens": 14362, "agent_thinking_tokens": 9753, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56541, "cached_input_tokens": 0, "completion_tokens": 1065, "thoughts_tokens": 2692, "total_tokens": 60298, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 38786, "cached_input_tokens": 14362, "completion_tokens": 350, "thoughts_tokens": 7061, "total_tokens": 46197, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [28.912], "agent_gross_standard_cost_usd": 0.0839713, "judge_gross_standard_cost_usd": 0.050178, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 123130, "agent_prompt_tokens": 117976, "agent_cached_input_tokens": 32240, "agent_thinking_tokens": 3562, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56321, "cached_input_tokens": 0, "completion_tokens": 1177, "thoughts_tokens": 2700, "total_tokens": 60198, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 61001, "cached_input_tokens": 32240, "completion_tokens": 359, "thoughts_tokens": 862, "total_tokens": 62222, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [27.769], "agent_gross_standard_cost_usd": 0.0855741, "judge_gross_standard_cost_usd": 0.0328365, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 123387, "agent_prompt_tokens": 117829, "agent_cached_input_tokens": 32230, "agent_thinking_tokens": 4084, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56292, "cached_input_tokens": 0, "completion_tokens": 1085, "thoughts_tokens": 3043, "total_tokens": 60420, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 60883, "cached_input_tokens": 32230, "completion_tokens": 333, "thoughts_tokens": 1041, "total_tokens": 62257, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [51.294], "agent_gross_standard_cost_usd": 0.13767279999999998, "judge_gross_standard_cost_usd": 0.0400275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 186693, "agent_prompt_tokens": 176242, "agent_cached_input_tokens": 37754, "agent_thinking_tokens": 7962, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 97239, "cached_input_tokens": 0, "completion_tokens": 1796, "thoughts_tokens": 6182, "total_tokens": 105217, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 78349, "cached_input_tokens": 37754, "completion_tokens": 637, "thoughts_tokens": 1780, "total_tokens": 80766, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [137.769], "agent_gross_standard_cost_usd": 0.2627404, "judge_gross_standard_cost_usd": 0.0570105, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 379692, "agent_prompt_tokens": 367192, "agent_cached_input_tokens": 214252, "agent_thinking_tokens": 10563, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 66402, "cached_input_tokens": 0, "completion_tokens": 1280, "thoughts_tokens": 3395, "total_tokens": 71077, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 300136, "cached_input_tokens": 214252, "completion_tokens": 601, "thoughts_tokens": 7168, "total_tokens": 307905, "request_count": 10 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 10, "tool_calls": 12, "main_agent_timed_model_calls": 10 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [23.414], "agent_gross_standard_cost_usd": 0.1434639, "judge_gross_standard_cost_usd": 0.034236, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 106282, "agent_prompt_tokens": 101856, "agent_cached_input_tokens": 18551, "agent_thinking_tokens": 2604, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56644, "cached_input_tokens": 0, "completion_tokens": 1334, "thoughts_tokens": 2604, "total_tokens": 60582, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 44558, "cached_input_tokens": 18551, "completion_tokens": 432, "thoughts_tokens": 0, "total_tokens": 44990, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 710, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [37.988], "agent_gross_standard_cost_usd": 0.18133559999999999, "judge_gross_standard_cost_usd": 0.046764, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 108801, "agent_prompt_tokens": 102515, "agent_cached_input_tokens": 18477, "agent_thinking_tokens": 4223, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56554, "cached_input_tokens": 0, "completion_tokens": 1494, "thoughts_tokens": 2423, "total_tokens": 60471, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1188, "cached_input_tokens": 0, "completion_tokens": 107, "thoughts_tokens": 0, "total_tokens": 1295, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 44773, "cached_input_tokens": 18477, "completion_tokens": 462, "thoughts_tokens": 1800, "total_tokens": 47035, "request_count": 2 } }, "agent_requests": 6, "main_agent_model_calls": 2, "tool_calls": 5, "main_agent_timed_model_calls": 2 }, { "task": "task-20", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 4.0, "customer_turn_seconds": [22.308], "agent_gross_standard_cost_usd": 0.0761004, "judge_gross_standard_cost_usd": 0.0410955, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 121593, "agent_prompt_tokens": 117741, "agent_cached_input_tokens": 32212, "agent_thinking_tokens": 2371, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 56347, "cached_input_tokens": 0, "completion_tokens": 1046, "thoughts_tokens": 2371, "total_tokens": 59764, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 60740, "cached_input_tokens": 32212, "completion_tokens": 369, "thoughts_tokens": 0, "total_tokens": 61109, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 654, "cached_input_tokens": 0, "completion_tokens": 66, "thoughts_tokens": 0, "total_tokens": 720, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.642], "agent_gross_standard_cost_usd": 0.17941469999999998, "judge_gross_standard_cost_usd": 0.024411, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 273434, "agent_prompt_tokens": 263098, "agent_cached_input_tokens": 96076, "agent_thinking_tokens": 7574, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 102891, "cached_input_tokens": 0, "completion_tokens": 1644, "thoughts_tokens": 6038, "total_tokens": 110573, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 159549, "cached_input_tokens": 96076, "completion_tokens": 1060, "thoughts_tokens": 1536, "total_tokens": 162145, "request_count": 7 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 11, "main_agent_model_calls": 7, "tool_calls": 7, "main_agent_timed_model_calls": 7 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [130.981], "agent_gross_standard_cost_usd": 0.09066937, "judge_gross_standard_cost_usd": 0.021435, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 103957, "agent_prompt_tokens": 98288, "agent_cached_input_tokens": 16329, "agent_thinking_tokens": 4363, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59084, "cached_input_tokens": 16329, "completion_tokens": 941, "thoughts_tokens": 3619, "total_tokens": 63644, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 38546, "cached_input_tokens": 0, "completion_tokens": 307, "thoughts_tokens": 744, "total_tokens": 39597, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [44.859], "agent_gross_standard_cost_usd": 0.08183285, "judge_gross_standard_cost_usd": 0.0211935, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 121260, "agent_prompt_tokens": 115414, "agent_cached_input_tokens": 31277, "agent_thinking_tokens": 4380, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59141, "cached_input_tokens": 0, "completion_tokens": 1151, "thoughts_tokens": 3380, "total_tokens": 63672, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 55615, "cached_input_tokens": 31277, "completion_tokens": 257, "thoughts_tokens": 1000, "total_tokens": 56872, "request_count": 3 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [196.976], "agent_gross_standard_cost_usd": 0.12317934999999999, "judge_gross_standard_cost_usd": 0.0234885, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155078, "agent_prompt_tokens": 146481, "agent_cached_input_tokens": 58737, "agent_thinking_tokens": 6724, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59406, "cached_input_tokens": 0, "completion_tokens": 1108, "thoughts_tokens": 2294, "total_tokens": 62808, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 86417, "cached_input_tokens": 58737, "completion_tokens": 712, "thoughts_tokens": 4430, "total_tokens": 91559, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 53, "thoughts_tokens": 0, "total_tokens": 711, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.226], "agent_gross_standard_cost_usd": 0.15222808000000002, "judge_gross_standard_cost_usd": 0.0225015, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 109566, "agent_prompt_tokens": 105206, "agent_cached_input_tokens": 26590, "agent_thinking_tokens": 2905, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59163, "cached_input_tokens": 10206, "completion_tokens": 1000, "thoughts_tokens": 2905, "total_tokens": 63068, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 45385, "cached_input_tokens": 16384, "completion_tokens": 397, "thoughts_tokens": 0, "total_tokens": 45782, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [116.573], "agent_gross_standard_cost_usd": 0.4782163, "judge_gross_standard_cost_usd": 0.0436485, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 227113, "agent_prompt_tokens": 208625, "agent_cached_input_tokens": 45632, "agent_thinking_tokens": 15345, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 111268, "cached_input_tokens": 0, "completion_tokens": 2135, "thoughts_tokens": 5717, "total_tokens": 119120, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 96699, "cached_input_tokens": 45632, "completion_tokens": 950, "thoughts_tokens": 9628, "total_tokens": 107277, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.188], "agent_gross_standard_cost_usd": 0.0851697, "judge_gross_standard_cost_usd": 0.021189, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 100152, "agent_prompt_tokens": 95023, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 3800, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59089, "cached_input_tokens": 0, "completion_tokens": 951, "thoughts_tokens": 3800, "total_tokens": 63840, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 35276, "cached_input_tokens": 0, "completion_tokens": 320, "thoughts_tokens": 0, "total_tokens": 35596, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 658, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 716, "request_count": 1 } }, "agent_requests": 5, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-21", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 1.0, "customer_turn_seconds": [153.04], "agent_gross_standard_cost_usd": 0.3025049, "judge_gross_standard_cost_usd": 0.032682, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 276999, "agent_prompt_tokens": 252345, "agent_cached_input_tokens": 60960, "agent_thinking_tokens": 22015, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 111568, "cached_input_tokens": 0, "completion_tokens": 1637, "thoughts_tokens": 7311, "total_tokens": 120516, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 140777, "cached_input_tokens": 60960, "completion_tokens": 1002, "thoughts_tokens": 14704, "total_tokens": 156483, "request_count": 6 } }, "agent_requests": 9, "main_agent_model_calls": 6, "tool_calls": 6, "main_agent_timed_model_calls": 6 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.157], "agent_gross_standard_cost_usd": 0.0492163, "judge_gross_standard_cost_usd": 0.0097665, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 66835, "agent_prompt_tokens": 63709, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2195, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48556, "cached_input_tokens": 0, "completion_tokens": 785, "thoughts_tokens": 1520, "total_tokens": 50861, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 15153, "cached_input_tokens": 0, "completion_tokens": 146, "thoughts_tokens": 675, "total_tokens": 15974, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [18.396], "agent_gross_standard_cost_usd": 0.0503388, "judge_gross_standard_cost_usd": 0.010089, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 67422, "agent_prompt_tokens": 63713, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2721, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48561, "cached_input_tokens": 0, "completion_tokens": 831, "thoughts_tokens": 2298, "total_tokens": 51690, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 15152, "cached_input_tokens": 0, "completion_tokens": 157, "thoughts_tokens": 423, "total_tokens": 15732, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.102], "agent_gross_standard_cost_usd": 0.0536236, "judge_gross_standard_cost_usd": 0.009936, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 67396, "agent_prompt_tokens": 63723, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2752, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48567, "cached_input_tokens": 0, "completion_tokens": 766, "thoughts_tokens": 1809, "total_tokens": 51142, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 15156, "cached_input_tokens": 0, "completion_tokens": 155, "thoughts_tokens": 943, "total_tokens": 16254, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [13.997], "agent_gross_standard_cost_usd": 0.0967954, "judge_gross_standard_cost_usd": 0.009519, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 69392, "agent_prompt_tokens": 66810, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 1654, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48558, "cached_input_tokens": 0, "completion_tokens": 770, "thoughts_tokens": 1654, "total_tokens": 50982, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 18252, "cached_input_tokens": 0, "completion_tokens": 158, "thoughts_tokens": 0, "total_tokens": 18410, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [20.22], "agent_gross_standard_cost_usd": 0.1130596, "judge_gross_standard_cost_usd": 0.0104085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 70249, "agent_prompt_tokens": 66820, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2495, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48567, "cached_input_tokens": 0, "completion_tokens": 806, "thoughts_tokens": 1657, "total_tokens": 51030, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 18253, "cached_input_tokens": 0, "completion_tokens": 128, "thoughts_tokens": 838, "total_tokens": 19219, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [16.636], "agent_gross_standard_cost_usd": 0.0489715, "judge_gross_standard_cost_usd": 0.009612, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 67129, "agent_prompt_tokens": 63723, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2417, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48565, "cached_input_tokens": 0, "completion_tokens": 834, "thoughts_tokens": 1942, "total_tokens": 51341, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 15158, "cached_input_tokens": 0, "completion_tokens": 155, "thoughts_tokens": 475, "total_tokens": 15788, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.367], "agent_gross_standard_cost_usd": 0.1065151, "judge_gross_standard_cost_usd": 0.0185865, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 115338, "agent_prompt_tokens": 109684, "agent_cached_input_tokens": 17798, "agent_thinking_tokens": 4689, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 53078, "cached_input_tokens": 0, "completion_tokens": 766, "thoughts_tokens": 1773, "total_tokens": 55617, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 56606, "cached_input_tokens": 17798, "completion_tokens": 199, "thoughts_tokens": 2916, "total_tokens": 59721, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-22", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [17.715], "agent_gross_standard_cost_usd": 0.0479792, "judge_gross_standard_cost_usd": 0.0097425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 66788, "agent_prompt_tokens": 63715, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2094, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 48559, "cached_input_tokens": 0, "completion_tokens": 825, "thoughts_tokens": 1649, "total_tokens": 51033, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 15156, "cached_input_tokens": 0, "completion_tokens": 154, "thoughts_tokens": 445, "total_tokens": 15755, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [191.887, 16.087], "agent_gross_standard_cost_usd": 0.227627325, "judge_gross_standard_cost_usd": 0.1422375, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 312813, "agent_prompt_tokens": 303853, "agent_cached_input_tokens": 23189, "agent_thinking_tokens": 5737, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 145382, "cached_input_tokens": 0, "completion_tokens": 1990, "thoughts_tokens": 4822, "total_tokens": 152194, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 107950, "cached_input_tokens": 14524, "completion_tokens": 520, "thoughts_tokens": 915, "total_tokens": 109385, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 50521, "cached_input_tokens": 8665, "completion_tokens": 713, "thoughts_tokens": 0, "total_tokens": 51234, "request_count": 5 } }, "agent_requests": 13, "main_agent_model_calls": 4, "tool_calls": 7, "main_agent_timed_model_calls": 4 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [95.089, 21.82], "agent_gross_standard_cost_usd": 0.26063715, "judge_gross_standard_cost_usd": 0.1323855, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 309602, "agent_prompt_tokens": 297592, "agent_cached_input_tokens": 21503, "agent_thinking_tokens": 9141, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 142994, "cached_input_tokens": 0, "completion_tokens": 1718, "thoughts_tokens": 3832, "total_tokens": 148544, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 106691, "cached_input_tokens": 14541, "completion_tokens": 472, "thoughts_tokens": 5309, "total_tokens": 112472, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 47907, "cached_input_tokens": 6962, "completion_tokens": 679, "thoughts_tokens": 0, "total_tokens": 48586, "request_count": 5 } }, "agent_requests": 13, "main_agent_model_calls": 4, "tool_calls": 7, "main_agent_timed_model_calls": 4 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.98, 13.507], "agent_gross_standard_cost_usd": 0.32845945, "judge_gross_standard_cost_usd": 0.069651, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 311211, "agent_prompt_tokens": 304295, "agent_cached_input_tokens": 92864, "agent_thinking_tokens": 4060, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 121805, "cached_input_tokens": 0, "completion_tokens": 1856, "thoughts_tokens": 4060, "total_tokens": 127721, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 138918, "cached_input_tokens": 78662, "completion_tokens": 604, "thoughts_tokens": 0, "total_tokens": 139522, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 43572, "cached_input_tokens": 14202, "completion_tokens": 396, "thoughts_tokens": 0, "total_tokens": 43968, "request_count": 5 } }, "agent_requests": 14, "main_agent_model_calls": 5, "tool_calls": 10, "main_agent_timed_model_calls": 5 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [86.672, 15.588], "agent_gross_standard_cost_usd": 0.6853406750000001, "judge_gross_standard_cost_usd": 0.114699, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 347756, "agent_prompt_tokens": 334416, "agent_cached_input_tokens": 52035, "agent_thinking_tokens": 10331, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 140165, "cached_input_tokens": 0, "completion_tokens": 1746, "thoughts_tokens": 4060, "total_tokens": 145971, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 163854, "cached_input_tokens": 45056, "completion_tokens": 607, "thoughts_tokens": 6271, "total_tokens": 170732, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 30397, "cached_input_tokens": 6979, "completion_tokens": 656, "thoughts_tokens": 0, "total_tokens": 31053, "request_count": 3 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 11, "main_agent_timed_model_calls": 5 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.776, 10.475], "agent_gross_standard_cost_usd": 0.203352715, "judge_gross_standard_cost_usd": 0.1649955, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 365490, "agent_prompt_tokens": 359877, "agent_cached_input_tokens": 92462, "agent_thinking_tokens": 2680, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 154455, "cached_input_tokens": 10138, "completion_tokens": 1696, "thoughts_tokens": 2680, "total_tokens": 158831, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 154960, "cached_input_tokens": 73661, "completion_tokens": 549, "thoughts_tokens": 0, "total_tokens": 155509, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 50462, "cached_input_tokens": 8663, "completion_tokens": 688, "thoughts_tokens": 0, "total_tokens": 51150, "request_count": 5 } }, "agent_requests": 14, "main_agent_model_calls": 5, "tool_calls": 9, "main_agent_timed_model_calls": 5 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [81.726, 17.801], "agent_gross_standard_cost_usd": 0.2383212, "judge_gross_standard_cost_usd": 0.1141275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 319443, "agent_prompt_tokens": 305867, "agent_cached_input_tokens": 72253, "agent_thinking_tokens": 10786, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 139830, "cached_input_tokens": 0, "completion_tokens": 1723, "thoughts_tokens": 3036, "total_tokens": 144589, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 135756, "cached_input_tokens": 65273, "completion_tokens": 539, "thoughts_tokens": 7750, "total_tokens": 144045, "request_count": 5 }, "gemini-3.1-flash-lite": { "prompt_tokens": 30281, "cached_input_tokens": 6980, "completion_tokens": 528, "thoughts_tokens": 0, "total_tokens": 30809, "request_count": 3 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 9, "main_agent_timed_model_calls": 5 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.256, 13.914], "agent_gross_standard_cost_usd": 0.188837435, "judge_gross_standard_cost_usd": 0.1370475, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 332884, "agent_prompt_tokens": 325852, "agent_cached_input_tokens": 75540, "agent_thinking_tokens": 3943, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 145142, "cached_input_tokens": 10137, "completion_tokens": 1790, "thoughts_tokens": 3814, "total_tokens": 150746, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 122271, "cached_input_tokens": 49142, "completion_tokens": 515, "thoughts_tokens": 129, "total_tokens": 122915, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 58439, "cached_input_tokens": 16261, "completion_tokens": 784, "thoughts_tokens": 0, "total_tokens": 59223, "request_count": 5 } }, "agent_requests": 13, "main_agent_model_calls": 4, "tool_calls": 10, "main_agent_timed_model_calls": 4 }, { "task": "task-23", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 1, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [149.099, 15.484], "agent_gross_standard_cost_usd": 0.1892059, "judge_gross_standard_cost_usd": 0.0927645, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 323773, "agent_prompt_tokens": 315355, "agent_cached_input_tokens": 92855, "agent_thinking_tokens": 5461, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 126958, "cached_input_tokens": 0, "completion_tokens": 1818, "thoughts_tokens": 3756, "total_tokens": 132532, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 158450, "cached_input_tokens": 92855, "completion_tokens": 480, "thoughts_tokens": 1705, "total_tokens": 160635, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 29947, "cached_input_tokens": 0, "completion_tokens": 659, "thoughts_tokens": 0, "total_tokens": 30606, "request_count": 4 } }, "agent_requests": 14, "main_agent_model_calls": 6, "tool_calls": 8, "main_agent_timed_model_calls": 6 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [90.297], "agent_gross_standard_cost_usd": 0.1082427, "judge_gross_standard_cost_usd": 0.02757, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155110, "agent_prompt_tokens": 148743, "agent_cached_input_tokens": 35885, "agent_thinking_tokens": 5154, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 83639, "cached_input_tokens": 0, "completion_tokens": 896, "thoughts_tokens": 2467, "total_tokens": 87002, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 903, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 953, "request_count": 1 }, "gemini-3.5-flash": { "prompt_tokens": 64201, "cached_input_tokens": 35885, "completion_tokens": 267, "thoughts_tokens": 2687, "total_tokens": 67155, "request_count": 3 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [47.11], "agent_gross_standard_cost_usd": 0.13452811, "judge_gross_standard_cost_usd": 0.012015, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 149351, "agent_prompt_tokens": 146782, "agent_cached_input_tokens": 60276, "agent_thinking_tokens": 1675, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 76764, "cached_input_tokens": 14207, "completion_tokens": 671, "thoughts_tokens": 1675, "total_tokens": 79110, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 70018, "cached_input_tokens": 46069, "completion_tokens": 223, "thoughts_tokens": 0, "total_tokens": 70241, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [38.968], "agent_gross_standard_cost_usd": 0.1850154, "judge_gross_standard_cost_usd": 0.0211275, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 129703, "agent_prompt_tokens": 124473, "agent_cached_input_tokens": 17114, "agent_thinking_tokens": 4095, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 83232, "cached_input_tokens": 0, "completion_tokens": 970, "thoughts_tokens": 1572, "total_tokens": 85774, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 41241, "cached_input_tokens": 17114, "completion_tokens": 165, "thoughts_tokens": 2523, "total_tokens": 43929, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.157], "agent_gross_standard_cost_usd": 0.08590170999999999, "judge_gross_standard_cost_usd": 0.02142, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 141045, "agent_prompt_tokens": 136465, "agent_cached_input_tokens": 40025, "agent_thinking_tokens": 3682, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 83038, "cached_input_tokens": 14207, "completion_tokens": 730, "thoughts_tokens": 2232, "total_tokens": 86000, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 53427, "cached_input_tokens": 25818, "completion_tokens": 168, "thoughts_tokens": 1450, "total_tokens": 55045, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [61.776], "agent_gross_standard_cost_usd": 0.14752111, "judge_gross_standard_cost_usd": 0.028908, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 216551, "agent_prompt_tokens": 207787, "agent_cached_input_tokens": 99681, "agent_thinking_tokens": 7588, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 84378, "cached_input_tokens": 14207, "completion_tokens": 847, "thoughts_tokens": 1035, "total_tokens": 86260, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 123409, "cached_input_tokens": 85474, "completion_tokens": 329, "thoughts_tokens": 6553, "total_tokens": 130291, "request_count": 6 } }, "agent_requests": 8, "main_agent_model_calls": 6, "tool_calls": 6, "main_agent_timed_model_calls": 6 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [24.091], "agent_gross_standard_cost_usd": 0.054697409999999995, "judge_gross_standard_cost_usd": 0.0102315, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 94825, "agent_prompt_tokens": 91526, "agent_cached_input_tokens": 14207, "agent_thinking_tokens": 2326, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 76436, "cached_input_tokens": 14207, "completion_tokens": 847, "thoughts_tokens": 1508, "total_tokens": 78791, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 15090, "cached_input_tokens": 0, "completion_tokens": 126, "thoughts_tokens": 818, "total_tokens": 16034, "request_count": 1 } }, "agent_requests": 3, "main_agent_model_calls": 1, "tool_calls": 0, "main_agent_timed_model_calls": 1 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.759], "agent_gross_standard_cost_usd": 0.06393950999999999, "judge_gross_standard_cost_usd": 0.0117135, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 110711, "agent_prompt_tokens": 108096, "agent_cached_input_tokens": 25089, "agent_thinking_tokens": 1760, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 76817, "cached_input_tokens": 14207, "completion_tokens": 708, "thoughts_tokens": 714, "total_tokens": 78239, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 31279, "cached_input_tokens": 10882, "completion_tokens": 147, "thoughts_tokens": 1046, "total_tokens": 32472, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 1, "main_agent_timed_model_calls": 2 }, { "task": "task-24", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "shopping/cross-sell", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [45.197], "agent_gross_standard_cost_usd": 0.06933876, "judge_gross_standard_cost_usd": 0.021075, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122853, "agent_prompt_tokens": 119330, "agent_cached_input_tokens": 29196, "agent_thinking_tokens": 2472, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 82696, "cached_input_tokens": 14207, "completion_tokens": 898, "thoughts_tokens": 1880, "total_tokens": 85474, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 36634, "cached_input_tokens": 14989, "completion_tokens": 153, "thoughts_tokens": 592, "total_tokens": 37379, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 1, "main_agent_timed_model_calls": 2 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.234], "agent_gross_standard_cost_usd": 0.1430486, "judge_gross_standard_cost_usd": 0.0140685, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 139213, "agent_prompt_tokens": 135810, "agent_cached_input_tokens": 57310, "agent_thinking_tokens": 2320, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52882, "cached_input_tokens": 0, "completion_tokens": 812, "thoughts_tokens": 2320, "total_tokens": 56014, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 82928, "cached_input_tokens": 57310, "completion_tokens": 271, "thoughts_tokens": 0, "total_tokens": 83199, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [45.155], "agent_gross_standard_cost_usd": 0.240401, "judge_gross_standard_cost_usd": 0.0149835, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 135310, "agent_prompt_tokens": 129669, "agent_cached_input_tokens": 38045, "agent_thinking_tokens": 4541, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52770, "cached_input_tokens": 0, "completion_tokens": 785, "thoughts_tokens": 2145, "total_tokens": 55700, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 76899, "cached_input_tokens": 38045, "completion_tokens": 315, "thoughts_tokens": 2396, "total_tokens": 79610, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 6, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.88], "agent_gross_standard_cost_usd": 0.08364575, "judge_gross_standard_cost_usd": 0.019299, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 152387, "agent_prompt_tokens": 148643, "agent_cached_input_tokens": 63341, "agent_thinking_tokens": 2634, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52682, "cached_input_tokens": 0, "completion_tokens": 812, "thoughts_tokens": 2634, "total_tokens": 56128, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 912, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 962, "request_count": 1 }, "gemini-3.8-flash": { "prompt_tokens": 95049, "cached_input_tokens": 63341, "completion_tokens": 248, "thoughts_tokens": 0, "total_tokens": 95297, "request_count": 5 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 5, "main_agent_timed_model_calls": 5 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [74.508], "agent_gross_standard_cost_usd": 0.11087235000000001, "judge_gross_standard_cost_usd": 0.0135225, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 116912, "agent_prompt_tokens": 107537, "agent_cached_input_tokens": 31751, "agent_thinking_tokens": 8457, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52064, "cached_input_tokens": 0, "completion_tokens": 705, "thoughts_tokens": 2376, "total_tokens": 55145, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 55473, "cached_input_tokens": 31751, "completion_tokens": 213, "thoughts_tokens": 6081, "total_tokens": 61767, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [124.539], "agent_gross_standard_cost_usd": 0.0864305, "judge_gross_standard_cost_usd": 0.012174, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 121471, "agent_prompt_tokens": 118697, "agent_cached_input_tokens": 27577, "agent_thinking_tokens": 1717, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50954, "cached_input_tokens": 0, "completion_tokens": 804, "thoughts_tokens": 1717, "total_tokens": 53475, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 913, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 963, "request_count": 1 }, "gemini-3.7-flash": { "prompt_tokens": 66830, "cached_input_tokens": 27577, "completion_tokens": 203, "thoughts_tokens": 0, "total_tokens": 67033, "request_count": 4 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 3, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [40.057], "agent_gross_standard_cost_usd": 0.0658051, "judge_gross_standard_cost_usd": 0.0111795, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 102854, "agent_prompt_tokens": 99056, "agent_cached_input_tokens": 27466, "agent_thinking_tokens": 2799, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50504, "cached_input_tokens": 0, "completion_tokens": 809, "thoughts_tokens": 1907, "total_tokens": 53220, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 48552, "cached_input_tokens": 27466, "completion_tokens": 190, "thoughts_tokens": 892, "total_tokens": 49634, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [285.906], "agent_gross_standard_cost_usd": 0.07283725, "judge_gross_standard_cost_usd": 0.0163995, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 124650, "agent_prompt_tokens": 120353, "agent_cached_input_tokens": 46561, "agent_thinking_tokens": 3363, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52847, "cached_input_tokens": 0, "completion_tokens": 720, "thoughts_tokens": 2371, "total_tokens": 55938, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 67506, "cached_input_tokens": 46561, "completion_tokens": 214, "thoughts_tokens": 992, "total_tokens": 68712, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-25", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [99.605], "agent_gross_standard_cost_usd": 0.2415805, "judge_gross_standard_cost_usd": 0.032529, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 203574, "agent_prompt_tokens": 196505, "agent_cached_input_tokens": 16608, "agent_thinking_tokens": 5940, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61076, "cached_input_tokens": 0, "completion_tokens": 836, "thoughts_tokens": 2408, "total_tokens": 64320, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 135429, "cached_input_tokens": 16608, "completion_tokens": 293, "thoughts_tokens": 3532, "total_tokens": 139254, "request_count": 5 } }, "agent_requests": 7, "main_agent_model_calls": 5, "tool_calls": 8, "main_agent_timed_model_calls": 5 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.466], "agent_gross_standard_cost_usd": 0.2829749, "judge_gross_standard_cost_usd": 0.024228, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 170605, "agent_prompt_tokens": 163004, "agent_cached_input_tokens": 63848, "agent_thinking_tokens": 6086, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52926, "cached_input_tokens": 0, "completion_tokens": 1097, "thoughts_tokens": 3366, "total_tokens": 57389, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 956, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 0, "total_tokens": 1006, "request_count": 1 }, "gemini-4-argon": { "prompt_tokens": 109122, "cached_input_tokens": 63848, "completion_tokens": 368, "thoughts_tokens": 2720, "total_tokens": 112210, "request_count": 5 } }, "agent_requests": 8, "main_agent_model_calls": 5, "tool_calls": 6, "main_agent_timed_model_calls": 5 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [20.541], "agent_gross_standard_cost_usd": 0.05565506, "judge_gross_standard_cost_usd": 0.0158835, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 104650, "agent_prompt_tokens": 101435, "agent_cached_input_tokens": 45910, "agent_thinking_tokens": 2230, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51074, "cached_input_tokens": 16317, "completion_tokens": 763, "thoughts_tokens": 2230, "total_tokens": 54067, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 50361, "cached_input_tokens": 29593, "completion_tokens": 222, "thoughts_tokens": 0, "total_tokens": 50583, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [189.723], "agent_gross_standard_cost_usd": 0.14567545, "judge_gross_standard_cost_usd": 0.02496, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 213104, "agent_prompt_tokens": 203347, "agent_cached_input_tokens": 110745, "agent_thinking_tokens": 8568, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 53439, "cached_input_tokens": 0, "completion_tokens": 875, "thoughts_tokens": 2903, "total_tokens": 57217, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 149908, "cached_input_tokens": 110745, "completion_tokens": 314, "thoughts_tokens": 5665, "total_tokens": 155887, "request_count": 7 } }, "agent_requests": 9, "main_agent_model_calls": 7, "tool_calls": 7, "main_agent_timed_model_calls": 7 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [57.747], "agent_gross_standard_cost_usd": 0.13631725, "judge_gross_standard_cost_usd": 0.026532, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 232959, "agent_prompt_tokens": 223965, "agent_cached_input_tokens": 47523, "agent_thinking_tokens": 6683, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 119996, "cached_input_tokens": 0, "completion_tokens": 1629, "thoughts_tokens": 5875, "total_tokens": 127500, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 87793, "cached_input_tokens": 47523, "completion_tokens": 483, "thoughts_tokens": 808, "total_tokens": 89084, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16176, "cached_input_tokens": 0, "completion_tokens": 199, "thoughts_tokens": 0, "total_tokens": 16375, "request_count": 2 } }, "agent_requests": 12, "main_agent_model_calls": 6, "tool_calls": 4, "main_agent_timed_model_calls": 6 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [55.831], "agent_gross_standard_cost_usd": 0.06732345, "judge_gross_standard_cost_usd": 0.0122835, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 103706, "agent_prompt_tokens": 98955, "agent_cached_input_tokens": 27393, "agent_thinking_tokens": 3635, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50830, "cached_input_tokens": 0, "completion_tokens": 917, "thoughts_tokens": 2836, "total_tokens": 54583, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 48125, "cached_input_tokens": 27393, "completion_tokens": 199, "thoughts_tokens": 799, "total_tokens": 49123, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [186.447], "agent_gross_standard_cost_usd": 0.07698785, "judge_gross_standard_cost_usd": 0.0198075, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 113338, "agent_prompt_tokens": 107827, "agent_cached_input_tokens": 35487, "agent_thinking_tokens": 4319, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 51291, "cached_input_tokens": 0, "completion_tokens": 921, "thoughts_tokens": 2909, "total_tokens": 55121, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 56536, "cached_input_tokens": 35487, "completion_tokens": 271, "thoughts_tokens": 1410, "total_tokens": 58217, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 4, "main_agent_timed_model_calls": 3 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [133.408], "agent_gross_standard_cost_usd": 0.11195655, "judge_gross_standard_cost_usd": 0.0216915, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 140758, "agent_prompt_tokens": 132219, "agent_cached_input_tokens": 56913, "agent_thinking_tokens": 7091, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 52102, "cached_input_tokens": 0, "completion_tokens": 1045, "thoughts_tokens": 2627, "total_tokens": 55774, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 80117, "cached_input_tokens": 56913, "completion_tokens": 403, "thoughts_tokens": 4464, "total_tokens": 84984, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-26", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [23.879], "agent_gross_standard_cost_usd": 0.1270001, "judge_gross_standard_cost_usd": 0.012984, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 111087, "agent_prompt_tokens": 108005, "agent_cached_input_tokens": 33676, "agent_thinking_tokens": 2025, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 50828, "cached_input_tokens": 0, "completion_tokens": 868, "thoughts_tokens": 2025, "total_tokens": 53721, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 57177, "cached_input_tokens": 33676, "completion_tokens": 189, "thoughts_tokens": 0, "total_tokens": 57366, "request_count": 3 } }, "agent_requests": 5, "main_agent_model_calls": 3, "tool_calls": 2, "main_agent_timed_model_calls": 3 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.128], "agent_gross_standard_cost_usd": 0.10026035, "judge_gross_standard_cost_usd": 0.029676, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122560, "agent_prompt_tokens": 118572, "agent_cached_input_tokens": 14531, "agent_thinking_tokens": 2438, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58934, "cached_input_tokens": 0, "completion_tokens": 1159, "thoughts_tokens": 2275, "total_tokens": 62368, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 59638, "cached_input_tokens": 14531, "completion_tokens": 391, "thoughts_tokens": 163, "total_tokens": 60192, "request_count": 4 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 1.0, "customer_turn_seconds": [107.38], "agent_gross_standard_cost_usd": 0.201813525, "judge_gross_standard_cost_usd": 0.0456795, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 216260, "agent_prompt_tokens": 200815, "agent_cached_input_tokens": 66411, "agent_thinking_tokens": 13327, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63645, "cached_input_tokens": 0, "completion_tokens": 1322, "thoughts_tokens": 3406, "total_tokens": 68373, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 109255, "cached_input_tokens": 55954, "completion_tokens": 388, "thoughts_tokens": 9921, "total_tokens": 119564, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 27915, "cached_input_tokens": 10457, "completion_tokens": 408, "thoughts_tokens": 0, "total_tokens": 28323, "request_count": 5 } }, "agent_requests": 11, "main_agent_model_calls": 4, "tool_calls": 10, "main_agent_timed_model_calls": 4 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [30.916], "agent_gross_standard_cost_usd": 0.07659774999999999, "judge_gross_standard_cost_usd": 0.028608, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 104186, "agent_prompt_tokens": 99342, "agent_cached_input_tokens": 16351, "agent_thinking_tokens": 3628, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58647, "cached_input_tokens": 0, "completion_tokens": 1005, "thoughts_tokens": 2254, "total_tokens": 61906, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 40695, "cached_input_tokens": 16351, "completion_tokens": 211, "thoughts_tokens": 1374, "total_tokens": 42280, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 2, "main_agent_timed_model_calls": 2 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [45.867], "agent_gross_standard_cost_usd": 0.11763145, "judge_gross_standard_cost_usd": 0.0309015, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 126612, "agent_prompt_tokens": 119991, "agent_cached_input_tokens": 14549, "agent_thinking_tokens": 5033, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58922, "cached_input_tokens": 0, "completion_tokens": 1217, "thoughts_tokens": 3116, "total_tokens": 63255, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 60397, "cached_input_tokens": 14549, "completion_tokens": 325, "thoughts_tokens": 1747, "total_tokens": 62469, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 46, "thoughts_tokens": 170, "total_tokens": 888, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [227.927], "agent_gross_standard_cost_usd": 0.11728245, "judge_gross_standard_cost_usd": 0.030987, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 126078, "agent_prompt_tokens": 119849, "agent_cached_input_tokens": 16421, "agent_thinking_tokens": 4690, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58966, "cached_input_tokens": 0, "completion_tokens": 1157, "thoughts_tokens": 2740, "total_tokens": 62863, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 60211, "cached_input_tokens": 16421, "completion_tokens": 334, "thoughts_tokens": 1802, "total_tokens": 62347, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 48, "thoughts_tokens": 148, "total_tokens": 868, "request_count": 1 } }, "agent_requests": 6, "main_agent_model_calls": 3, "tool_calls": 3, "main_agent_timed_model_calls": 3 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [164.865], "agent_gross_standard_cost_usd": 0.30242215, "judge_gross_standard_cost_usd": 0.0367965, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 251125, "agent_prompt_tokens": 234742, "agent_cached_input_tokens": 16431, "agent_thinking_tokens": 13846, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 110645, "cached_input_tokens": 0, "completion_tokens": 1798, "thoughts_tokens": 4626, "total_tokens": 117069, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 123425, "cached_input_tokens": 16431, "completion_tokens": 683, "thoughts_tokens": 8992, "total_tokens": 133100, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 228, "total_tokens": 956, "request_count": 1 } }, "agent_requests": 9, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [39.562], "agent_gross_standard_cost_usd": 0.2845495, "judge_gross_standard_cost_usd": 0.039801, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 176593, "agent_prompt_tokens": 171656, "agent_cached_input_tokens": 49015, "agent_thinking_tokens": 3087, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63725, "cached_input_tokens": 0, "completion_tokens": 1318, "thoughts_tokens": 2931, "total_tokens": 67974, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 107259, "cached_input_tokens": 49015, "completion_tokens": 469, "thoughts_tokens": 0, "total_tokens": 107728, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 672, "cached_input_tokens": 0, "completion_tokens": 63, "thoughts_tokens": 156, "total_tokens": 891, "request_count": 1 } }, "agent_requests": 7, "main_agent_model_calls": 4, "tool_calls": 5, "main_agent_timed_model_calls": 4 }, { "task": "task-27", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "returns/exchanges", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [59.009], "agent_gross_standard_cost_usd": 0.2465404, "judge_gross_standard_cost_usd": 0.0380115, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122200, "agent_prompt_tokens": 114150, "agent_cached_input_tokens": 18502, "agent_thinking_tokens": 6564, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62985, "cached_input_tokens": 0, "completion_tokens": 1198, "thoughts_tokens": 2671, "total_tokens": 66854, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 51165, "cached_input_tokens": 18502, "completion_tokens": 288, "thoughts_tokens": 3893, "total_tokens": 55346, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 2, "tool_calls": 3, "main_agent_timed_model_calls": 2 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [46.824], "agent_gross_standard_cost_usd": 0.11318145, "judge_gross_standard_cost_usd": 0.033642, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 144932, "agent_prompt_tokens": 137157, "agent_cached_input_tokens": 39745, "agent_thinking_tokens": 6159, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63834, "cached_input_tokens": 0, "completion_tokens": 1114, "thoughts_tokens": 2502, "total_tokens": 67450, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 71827, "cached_input_tokens": 39745, "completion_tokens": 392, "thoughts_tokens": 3657, "total_tokens": 75876, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 8, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [25.244], "agent_gross_standard_cost_usd": 0.0697827, "judge_gross_standard_cost_usd": 0.0222735, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 99926, "agent_prompt_tokens": 95949, "agent_cached_input_tokens": 13958, "agent_thinking_tokens": 2607, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57300, "cached_input_tokens": 0, "completion_tokens": 1016, "thoughts_tokens": 1751, "total_tokens": 60067, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 37153, "cached_input_tokens": 13958, "completion_tokens": 244, "thoughts_tokens": 856, "total_tokens": 38253, "request_count": 2 } }, "agent_requests": 6, "main_agent_model_calls": 2, "tool_calls": 4, "main_agent_timed_model_calls": 2 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [76.818], "agent_gross_standard_cost_usd": 0.0936977, "judge_gross_standard_cost_usd": 0.028989, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 131872, "agent_prompt_tokens": 125295, "agent_cached_input_tokens": 31838, "agent_thinking_tokens": 5074, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 61580, "cached_input_tokens": 0, "completion_tokens": 1069, "thoughts_tokens": 3764, "total_tokens": 66413, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.7-flash": { "prompt_tokens": 62219, "cached_input_tokens": 31838, "completion_tokens": 324, "thoughts_tokens": 1310, "total_tokens": 63853, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [113.114], "agent_gross_standard_cost_usd": 0.12735745, "judge_gross_standard_cost_usd": 0.030285, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 203535, "agent_prompt_tokens": 196394, "agent_cached_input_tokens": 96529, "agent_thinking_tokens": 5116, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57822, "cached_input_tokens": 0, "completion_tokens": 1164, "thoughts_tokens": 3314, "total_tokens": 62300, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 137076, "cached_input_tokens": 96529, "completion_tokens": 751, "thoughts_tokens": 1802, "total_tokens": 139629, "request_count": 7 } }, "agent_requests": 11, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [212.105], "agent_gross_standard_cost_usd": 0.1134278, "judge_gross_standard_cost_usd": 0.0280755, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 198774, "agent_prompt_tokens": 192197, "agent_cached_input_tokens": 100962, "agent_thinking_tokens": 4958, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 59690, "cached_input_tokens": 0, "completion_tokens": 1101, "thoughts_tokens": 2505, "total_tokens": 63296, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 131011, "cached_input_tokens": 100962, "completion_tokens": 408, "thoughts_tokens": 2453, "total_tokens": 133872, "request_count": 6 } }, "agent_requests": 10, "main_agent_model_calls": 6, "tool_calls": 8, "main_agent_timed_model_calls": 6 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.688], "agent_gross_standard_cost_usd": 0.1503989, "judge_gross_standard_cost_usd": 0.023556, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 131933, "agent_prompt_tokens": 127884, "agent_cached_input_tokens": 41891, "agent_thinking_tokens": 2257, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 57824, "cached_input_tokens": 0, "completion_tokens": 1248, "thoughts_tokens": 2257, "total_tokens": 61329, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 68564, "cached_input_tokens": 41891, "completion_tokens": 434, "thoughts_tokens": 0, "total_tokens": 68998, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [44.392], "agent_gross_standard_cost_usd": 0.2291976, "judge_gross_standard_cost_usd": 0.037059, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 156046, "agent_prompt_tokens": 149136, "agent_cached_input_tokens": 48211, "agent_thinking_tokens": 4897, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 65033, "cached_input_tokens": 0, "completion_tokens": 1295, "thoughts_tokens": 2938, "total_tokens": 69266, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-4-argon": { "prompt_tokens": 82607, "cached_input_tokens": 48211, "completion_tokens": 608, "thoughts_tokens": 1959, "total_tokens": 85174, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 9, "main_agent_timed_model_calls": 3 }, { "task": "task-28", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.474], "agent_gross_standard_cost_usd": 0.0893819, "judge_gross_standard_cost_usd": 0.0327555, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 155480, "agent_prompt_tokens": 151297, "agent_cached_input_tokens": 53110, "agent_thinking_tokens": 2468, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 63548, "cached_input_tokens": 0, "completion_tokens": 1222, "thoughts_tokens": 2468, "total_tokens": 67238, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1496, "cached_input_tokens": 0, "completion_tokens": 110, "thoughts_tokens": 0, "total_tokens": 1606, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 86253, "cached_input_tokens": 53110, "completion_tokens": 383, "thoughts_tokens": 0, "total_tokens": 86636, "request_count": 4 } }, "agent_requests": 8, "main_agent_model_calls": 4, "tool_calls": 7, "main_agent_timed_model_calls": 4 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [28.48, 24.013, 41.585], "agent_gross_standard_cost_usd": 0.195462, "judge_gross_standard_cost_usd": 0.0248565, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 368056, "agent_prompt_tokens": 357341, "agent_cached_input_tokens": 113663, "agent_thinking_tokens": 6681, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 182386, "cached_input_tokens": 0, "completion_tokens": 3166, "thoughts_tokens": 5854, "total_tokens": 191406, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1987, "cached_input_tokens": 0, "completion_tokens": 170, "thoughts_tokens": 0, "total_tokens": 2157, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 172968, "cached_input_tokens": 113663, "completion_tokens": 698, "thoughts_tokens": 827, "total_tokens": 174493, "request_count": 9 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 7, "main_agent_timed_model_calls": 9 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.389, 40.334, 21.591], "agent_gross_standard_cost_usd": 0.20290135, "judge_gross_standard_cost_usd": 0.0288795, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 356070, "agent_prompt_tokens": 342955, "agent_cached_input_tokens": 100520, "agent_thinking_tokens": 9203, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 182017, "cached_input_tokens": 0, "completion_tokens": 3049, "thoughts_tokens": 7434, "total_tokens": 192500, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1989, "cached_input_tokens": 0, "completion_tokens": 145, "thoughts_tokens": 0, "total_tokens": 2134, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 158949, "cached_input_tokens": 100520, "completion_tokens": 718, "thoughts_tokens": 1769, "total_tokens": 161436, "request_count": 8 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 7, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [93.681, 25.645, 25.606], "agent_gross_standard_cost_usd": 0.18360315, "judge_gross_standard_cost_usd": 0.0206175, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 333370, "agent_prompt_tokens": 323546, "agent_cached_input_tokens": 94425, "agent_thinking_tokens": 6123, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180205, "cached_input_tokens": 10130, "completion_tokens": 2864, "thoughts_tokens": 5381, "total_tokens": 188450, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1988, "cached_input_tokens": 0, "completion_tokens": 170, "thoughts_tokens": 0, "total_tokens": 2158, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 141353, "cached_input_tokens": 84295, "completion_tokens": 667, "thoughts_tokens": 742, "total_tokens": 142762, "request_count": 8 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 6, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [353.013, 67.691, 90.272], "agent_gross_standard_cost_usd": 0.19529765, "judge_gross_standard_cost_usd": 0.0230145, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 343843, "agent_prompt_tokens": 331931, "agent_cached_input_tokens": 101548, "agent_thinking_tokens": 8145, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 180344, "cached_input_tokens": 0, "completion_tokens": 2894, "thoughts_tokens": 5113, "total_tokens": 188351, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 2237, "cached_input_tokens": 0, "completion_tokens": 195, "thoughts_tokens": 0, "total_tokens": 2432, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 149350, "cached_input_tokens": 101548, "completion_tokens": 678, "thoughts_tokens": 3032, "total_tokens": 153060, "request_count": 8 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 5, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.331, 24.238, 18.61], "agent_gross_standard_cost_usd": 0.4590192, "judge_gross_standard_cost_usd": 0.031878, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 384785, "agent_prompt_tokens": 374987, "agent_cached_input_tokens": 102622, "agent_thinking_tokens": 5517, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 184121, "cached_input_tokens": 0, "completion_tokens": 3334, "thoughts_tokens": 5517, "total_tokens": 192972, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1988, "cached_input_tokens": 0, "completion_tokens": 180, "thoughts_tokens": 0, "total_tokens": 2168, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 188878, "cached_input_tokens": 102622, "completion_tokens": 767, "thoughts_tokens": 0, "total_tokens": 189645, "request_count": 8 } }, "agent_requests": 17, "main_agent_model_calls": 8, "tool_calls": 8, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 1.0, "customer_turn_seconds": [41.046, 80.146, 0.003], "agent_gross_standard_cost_usd": 0.6273972, "judge_gross_standard_cost_usd": 0.0403005, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 463961, "agent_prompt_tokens": 446328, "agent_cached_input_tokens": 132100, "agent_thinking_tokens": 12920, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 200939, "cached_input_tokens": 0, "completion_tokens": 3362, "thoughts_tokens": 8150, "total_tokens": 212451, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 15506, "cached_input_tokens": 0, "completion_tokens": 298, "thoughts_tokens": 0, "total_tokens": 15804, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 229883, "cached_input_tokens": 132100, "completion_tokens": 1053, "thoughts_tokens": 4770, "total_tokens": 235706, "request_count": 9 } }, "agent_requests": 19, "main_agent_model_calls": 9, "tool_calls": 12, "main_agent_timed_model_calls": 9 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [31.748, 65.209, 0.003], "agent_gross_standard_cost_usd": 0.2024635, "judge_gross_standard_cost_usd": 0.0205245, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 382803, "agent_prompt_tokens": 369823, "agent_cached_input_tokens": 98219, "agent_thinking_tokens": 9153, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 196943, "cached_input_tokens": 0, "completion_tokens": 2794, "thoughts_tokens": 9153, "total_tokens": 208890, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 15327, "cached_input_tokens": 0, "completion_tokens": 300, "thoughts_tokens": 0, "total_tokens": 15627, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 157553, "cached_input_tokens": 98219, "completion_tokens": 733, "thoughts_tokens": 0, "total_tokens": 158286, "request_count": 8 } }, "agent_requests": 18, "main_agent_model_calls": 8, "tool_calls": 6, "main_agent_timed_model_calls": 8 }, { "task": "task-29", "suite": "Customer reply quality", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "hallucination/policy", "support/escalation", "tone/transcript", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [68.35, 34.485, 26.135], "agent_gross_standard_cost_usd": 0.22303109999999998, "judge_gross_standard_cost_usd": 0.029268, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 379897, "agent_prompt_tokens": 367469, "agent_cached_input_tokens": 119806, "agent_thinking_tokens": 8829, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 183014, "cached_input_tokens": 0, "completion_tokens": 2703, "thoughts_tokens": 4520, "total_tokens": 190237, "request_count": 6 }, "gemini-3.1-flash-lite": { "prompt_tokens": 2266, "cached_input_tokens": 0, "completion_tokens": 180, "thoughts_tokens": 0, "total_tokens": 2446, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 182189, "cached_input_tokens": 119806, "completion_tokens": 716, "thoughts_tokens": 4309, "total_tokens": 187214, "request_count": 9 } }, "agent_requests": 18, "main_agent_model_calls": 9, "tool_calls": 8, "main_agent_timed_model_calls": 9 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [14.138], "agent_gross_standard_cost_usd": 0.01643958, "judge_gross_standard_cost_usd": 0.0120975, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48118, "agent_prompt_tokens": 44979, "agent_cached_input_tokens": 16286, "agent_thinking_tokens": 2046, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39131, "cached_input_tokens": 16286, "completion_tokens": 881, "thoughts_tokens": 2046, "total_tokens": 42058, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 212, "thoughts_tokens": 0, "total_tokens": 6060, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [14.892], "agent_gross_standard_cost_usd": 0.020628599999999997, "judge_gross_standard_cost_usd": 0.012594, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48391, "agent_prompt_tokens": 44976, "agent_cached_input_tokens": 3988, "agent_thinking_tokens": 2208, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39128, "cached_input_tokens": 0, "completion_tokens": 995, "thoughts_tokens": 2208, "total_tokens": 42331, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 3988, "completion_tokens": 212, "thoughts_tokens": 0, "total_tokens": 6060, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [16.062], "agent_gross_standard_cost_usd": 0.023676, "judge_gross_standard_cost_usd": 0.012246, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 49262, "agent_prompt_tokens": 44988, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 3112, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39140, "cached_input_tokens": 0, "completion_tokens": 949, "thoughts_tokens": 3112, "total_tokens": 43201, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 213, "thoughts_tokens": 0, "total_tokens": 6061, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [14.834], "agent_gross_standard_cost_usd": 0.021264999999999996, "judge_gross_standard_cost_usd": 0.013653, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48661, "agent_prompt_tokens": 44994, "agent_cached_input_tokens": 3988, "agent_thinking_tokens": 2331, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39146, "cached_input_tokens": 0, "completion_tokens": 1125, "thoughts_tokens": 2331, "total_tokens": 42602, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 3988, "completion_tokens": 211, "thoughts_tokens": 0, "total_tokens": 6059, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [14.873], "agent_gross_standard_cost_usd": 0.022091700000000002, "judge_gross_standard_cost_usd": 0.0113895, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 48615, "agent_prompt_tokens": 44977, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 2378, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39129, "cached_input_tokens": 0, "completion_tokens": 1056, "thoughts_tokens": 2378, "total_tokens": 42563, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 204, "thoughts_tokens": 0, "total_tokens": 6052, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [12.502], "agent_gross_standard_cost_usd": 0.0193196, "judge_gross_standard_cost_usd": 0.011637, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 47889, "agent_prompt_tokens": 45001, "agent_cached_input_tokens": 3988, "agent_thinking_tokens": 1785, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39153, "cached_input_tokens": 0, "completion_tokens": 892, "thoughts_tokens": 1785, "total_tokens": 41830, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 3988, "completion_tokens": 211, "thoughts_tokens": 0, "total_tokens": 6059, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [16.107], "agent_gross_standard_cost_usd": 0.022858499999999997, "judge_gross_standard_cost_usd": 0.0107385, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 49290, "agent_prompt_tokens": 44984, "agent_cached_input_tokens": 3988, "agent_thinking_tokens": 3194, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39136, "cached_input_tokens": 0, "completion_tokens": 900, "thoughts_tokens": 3194, "total_tokens": 43230, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 3988, "completion_tokens": 212, "thoughts_tokens": 0, "total_tokens": 6060, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-30", "suite": "Support hours", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "support/escalation", "tone/transcript"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [12.945], "agent_gross_standard_cost_usd": 0.020275599999999998, "judge_gross_standard_cost_usd": 0.0112905, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 47916, "agent_prompt_tokens": 45005, "agent_cached_input_tokens": 0, "agent_thinking_tokens": 1651, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 39157, "cached_input_tokens": 0, "completion_tokens": 1049, "thoughts_tokens": 1651, "total_tokens": 41857, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 5848, "cached_input_tokens": 0, "completion_tokens": 211, "thoughts_tokens": 0, "total_tokens": 6059, "request_count": 2 } }, "agent_requests": 4, "main_agent_model_calls": 0, "tool_calls": 0, "main_agent_timed_model_calls": 0 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [154.863], "agent_gross_standard_cost_usd": 0.083211785, "judge_gross_standard_cost_usd": 0.055425, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 145256, "agent_prompt_tokens": 140693, "agent_cached_input_tokens": 59155, "agent_thinking_tokens": 3139, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60902, "cached_input_tokens": 16332, "completion_tokens": 968, "thoughts_tokens": 2313, "total_tokens": 64183, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 13038, "cached_input_tokens": 5031, "completion_tokens": 217, "thoughts_tokens": 0, "total_tokens": 13255, "request_count": 3 }, "gemini-3.5-flash": { "prompt_tokens": 66753, "cached_input_tokens": 37792, "completion_tokens": 239, "thoughts_tokens": 826, "total_tokens": 67818, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [161.557], "agent_gross_standard_cost_usd": 0.10164075, "judge_gross_standard_cost_usd": 0.045315, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 139511, "agent_prompt_tokens": 133182, "agent_cached_input_tokens": 37959, "agent_thinking_tokens": 4715, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 58613, "cached_input_tokens": 0, "completion_tokens": 1128, "thoughts_tokens": 2174, "total_tokens": 61915, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 8518, "cached_input_tokens": 0, "completion_tokens": 187, "thoughts_tokens": 0, "total_tokens": 8705, "request_count": 2 }, "gemini-3.5-flash": { "prompt_tokens": 66051, "cached_input_tokens": 37959, "completion_tokens": 299, "thoughts_tokens": 2541, "total_tokens": 68891, "request_count": 3 } }, "agent_requests": 7, "main_agent_model_calls": 3, "tool_calls": 5, "main_agent_timed_model_calls": 3 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.82], "agent_gross_standard_cost_usd": 0.2042495, "judge_gross_standard_cost_usd": 0.0617895, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 200399, "agent_prompt_tokens": 195966, "agent_cached_input_tokens": 78806, "agent_thinking_tokens": 2710, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62236, "cached_input_tokens": 0, "completion_tokens": 1072, "thoughts_tokens": 2710, "total_tokens": 66018, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 18130, "cached_input_tokens": 0, "completion_tokens": 236, "thoughts_tokens": 0, "total_tokens": 18366, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 115600, "cached_input_tokens": 78806, "completion_tokens": 415, "thoughts_tokens": 0, "total_tokens": 116015, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 9, "main_agent_timed_model_calls": 4 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [64.195], "agent_gross_standard_cost_usd": 0.36072555, "judge_gross_standard_cost_usd": 0.0610455, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 200735, "agent_prompt_tokens": 192896, "agent_cached_input_tokens": 58306, "agent_thinking_tokens": 6191, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 62337, "cached_input_tokens": 0, "completion_tokens": 916, "thoughts_tokens": 2973, "total_tokens": 66226, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 18194, "cached_input_tokens": 7050, "completion_tokens": 235, "thoughts_tokens": 0, "total_tokens": 18429, "request_count": 3 }, "gemini-4-argon": { "prompt_tokens": 112365, "cached_input_tokens": 51256, "completion_tokens": 497, "thoughts_tokens": 3218, "total_tokens": 116080, "request_count": 4 } }, "agent_requests": 9, "main_agent_model_calls": 4, "tool_calls": 10, "main_agent_timed_model_calls": 4 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [27.07], "agent_gross_standard_cost_usd": 0.077782875, "judge_gross_standard_cost_usd": 0.0562905, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 149299, "agent_prompt_tokens": 145610, "agent_cached_input_tokens": 46882, "agent_thinking_tokens": 2244, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60894, "cached_input_tokens": 0, "completion_tokens": 943, "thoughts_tokens": 2244, "total_tokens": 64081, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 18140, "cached_input_tokens": 7047, "completion_tokens": 209, "thoughts_tokens": 0, "total_tokens": 18349, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 66576, "cached_input_tokens": 39835, "completion_tokens": 293, "thoughts_tokens": 0, "total_tokens": 66869, "request_count": 3 } }, "agent_requests": 8, "main_agent_model_calls": 3, "tool_calls": 6, "main_agent_timed_model_calls": 3 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [90.706], "agent_gross_standard_cost_usd": 0.1724674, "judge_gross_standard_cost_usd": 0.049989, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 212931, "agent_prompt_tokens": 200292, "agent_cached_input_tokens": 91034, "agent_thinking_tokens": 11077, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60106, "cached_input_tokens": 0, "completion_tokens": 999, "thoughts_tokens": 2254, "total_tokens": 63359, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 8532, "cached_input_tokens": 0, "completion_tokens": 135, "thoughts_tokens": 0, "total_tokens": 8667, "request_count": 2 }, "gemini-3.8-flash": { "prompt_tokens": 131654, "cached_input_tokens": 91034, "completion_tokens": 428, "thoughts_tokens": 8823, "total_tokens": 140905, "request_count": 5 } }, "agent_requests": 9, "main_agent_model_calls": 5, "tool_calls": 9, "main_agent_timed_model_calls": 5 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.599], "agent_gross_standard_cost_usd": 0.07874985, "judge_gross_standard_cost_usd": 0.056679, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 120484, "agent_prompt_tokens": 116634, "agent_cached_input_tokens": 19316, "agent_thinking_tokens": 2256, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60909, "cached_input_tokens": 0, "completion_tokens": 1131, "thoughts_tokens": 1661, "total_tokens": 63701, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 13030, "cached_input_tokens": 5030, "completion_tokens": 220, "thoughts_tokens": 0, "total_tokens": 13250, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 42695, "cached_input_tokens": 14286, "completion_tokens": 243, "thoughts_tokens": 595, "total_tokens": 43533, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-31", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": [ "catalog grounding", "hallucination/policy", "shopping/cross-sell", "tool-use correctness" ], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [31.429], "agent_gross_standard_cost_usd": 0.084525875, "judge_gross_standard_cost_usd": 0.057696, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 122177, "agent_prompt_tokens": 116656, "agent_cached_input_tokens": 19316, "agent_thinking_tokens": 4025, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 60927, "cached_input_tokens": 0, "completion_tokens": 1021, "thoughts_tokens": 3124, "total_tokens": 65072, "request_count": 2 }, "gemini-3.1-flash-lite": { "prompt_tokens": 13038, "cached_input_tokens": 5031, "completion_tokens": 219, "thoughts_tokens": 0, "total_tokens": 13257, "request_count": 3 }, "gemini-3.7-flash": { "prompt_tokens": 42691, "cached_input_tokens": 14285, "completion_tokens": 256, "thoughts_tokens": 901, "total_tokens": 43848, "request_count": 2 } }, "agent_requests": 7, "main_agent_model_calls": 2, "tool_calls": 6, "main_agent_timed_model_calls": 2 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [53.859, 115.025], "agent_gross_standard_cost_usd": 0.2891582, "judge_gross_standard_cost_usd": 0.110679, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 412282, "agent_prompt_tokens": 398333, "agent_cached_input_tokens": 152326, "agent_thinking_tokens": 10124, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 127256, "cached_input_tokens": 0, "completion_tokens": 2480, "thoughts_tokens": 3089, "total_tokens": 132825, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 43405, "cached_input_tokens": 15842, "completion_tokens": 387, "thoughts_tokens": 0, "total_tokens": 43792, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 227672, "cached_input_tokens": 136484, "completion_tokens": 958, "thoughts_tokens": 7035, "total_tokens": 235665, "request_count": 9 } }, "agent_requests": 19, "main_agent_model_calls": 9, "tool_calls": 16, "main_agent_timed_model_calls": 9 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [30.944, 27.924], "agent_gross_standard_cost_usd": 0.4923484, "judge_gross_standard_cost_usd": 0.1181775, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 331746, "agent_prompt_tokens": 323849, "agent_cached_input_tokens": 77845, "agent_thinking_tokens": 4603, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 126744, "cached_input_tokens": 0, "completion_tokens": 2299, "thoughts_tokens": 4603, "total_tokens": 133646, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25517, "cached_input_tokens": 7046, "completion_tokens": 267, "thoughts_tokens": 0, "total_tokens": 25784, "request_count": 4 }, "gemini-4-argon": { "prompt_tokens": 171588, "cached_input_tokens": 70799, "completion_tokens": 728, "thoughts_tokens": 0, "total_tokens": 172316, "request_count": 6 } }, "agent_requests": 14, "main_agent_model_calls": 6, "tool_calls": 14, "main_agent_timed_model_calls": 6 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [68.047, 49.737], "agent_gross_standard_cost_usd": 0.5692233, "judge_gross_standard_cost_usd": 0.1206405, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 447940, "agent_prompt_tokens": 434090, "agent_cached_input_tokens": 183742, "agent_thinking_tokens": 10107, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 131265, "cached_input_tokens": 0, "completion_tokens": 2237, "thoughts_tokens": 4050, "total_tokens": 137552, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 44828, "cached_input_tokens": 7152, "completion_tokens": 415, "thoughts_tokens": 0, "total_tokens": 45243, "request_count": 7 }, "gemini-4-argon": { "prompt_tokens": 257997, "cached_input_tokens": 176590, "completion_tokens": 1091, "thoughts_tokens": 6057, "total_tokens": 265145, "request_count": 8 } }, "agent_requests": 19, "main_agent_model_calls": 8, "tool_calls": 21, "main_agent_timed_model_calls": 8 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [33.52, 27.768], "agent_gross_standard_cost_usd": 0.19250254, "judge_gross_standard_cost_usd": 0.096471, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 362437, "agent_prompt_tokens": 354592, "agent_cached_input_tokens": 145848, "agent_thinking_tokens": 4816, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 120324, "cached_input_tokens": 10133, "completion_tokens": 2048, "thoughts_tokens": 4816, "total_tokens": 127188, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25516, "cached_input_tokens": 0, "completion_tokens": 278, "thoughts_tokens": 0, "total_tokens": 25794, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 208752, "cached_input_tokens": 135715, "completion_tokens": 703, "thoughts_tokens": 0, "total_tokens": 209455, "request_count": 9 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 13, "main_agent_timed_model_calls": 9 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [93.311, 51.238], "agent_gross_standard_cost_usd": 0.27697065, "judge_gross_standard_cost_usd": 0.104805, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 447406, "agent_prompt_tokens": 433299, "agent_cached_input_tokens": 199768, "agent_thinking_tokens": 10838, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 128364, "cached_input_tokens": 0, "completion_tokens": 2141, "thoughts_tokens": 3724, "total_tokens": 134229, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 25554, "cached_input_tokens": 7046, "completion_tokens": 286, "thoughts_tokens": 0, "total_tokens": 25840, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 279381, "cached_input_tokens": 192722, "completion_tokens": 842, "thoughts_tokens": 7114, "total_tokens": 287337, "request_count": 10 } }, "agent_requests": 18, "main_agent_model_calls": 10, "tool_calls": 17, "main_agent_timed_model_calls": 10 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [47.608, 27.912], "agent_gross_standard_cost_usd": 0.1498399, "judge_gross_standard_cost_usd": 0.0860985, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 259987, "agent_prompt_tokens": 251163, "agent_cached_input_tokens": 59076, "agent_thinking_tokens": 5548, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 119125, "cached_input_tokens": 0, "completion_tokens": 2456, "thoughts_tokens": 5548, "total_tokens": 127129, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 22960, "cached_input_tokens": 0, "completion_tokens": 277, "thoughts_tokens": 0, "total_tokens": 23237, "request_count": 4 }, "gemini-3.7-flash": { "prompt_tokens": 109078, "cached_input_tokens": 59076, "completion_tokens": 543, "thoughts_tokens": 0, "total_tokens": 109621, "request_count": 5 } }, "agent_requests": 13, "main_agent_model_calls": 5, "tool_calls": 10, "main_agent_timed_model_calls": 5 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [80.583, 36.004], "agent_gross_standard_cost_usd": 0.20579214999999998, "judge_gross_standard_cost_usd": 0.104886, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 343675, "agent_prompt_tokens": 331916, "agent_cached_input_tokens": 113402, "agent_thinking_tokens": 8656, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 127052, "cached_input_tokens": 0, "completion_tokens": 2102, "thoughts_tokens": 5639, "total_tokens": 134793, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 33732, "cached_input_tokens": 7046, "completion_tokens": 296, "thoughts_tokens": 0, "total_tokens": 34028, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 171132, "cached_input_tokens": 106356, "completion_tokens": 705, "thoughts_tokens": 3017, "total_tokens": 174854, "request_count": 7 } }, "agent_requests": 16, "main_agent_model_calls": 7, "tool_calls": 13, "main_agent_timed_model_calls": 7 }, { "task": "task-32", "suite": "Purchase assistance", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["catalog grounding", "shopping/cross-sell", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 3.0, "customer_turn_seconds": [134.52, 83.156], "agent_gross_standard_cost_usd": 0.2208156, "judge_gross_standard_cost_usd": 0.092874, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 332593, "agent_prompt_tokens": 322905, "agent_cached_input_tokens": 101536, "agent_thinking_tokens": 6461, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 120564, "cached_input_tokens": 0, "completion_tokens": 2237, "thoughts_tokens": 4518, "total_tokens": 127319, "request_count": 4 }, "gemini-3.1-flash-lite": { "prompt_tokens": 23026, "cached_input_tokens": 5268, "completion_tokens": 296, "thoughts_tokens": 0, "total_tokens": 23322, "request_count": 4 }, "gemini-3.5-flash": { "prompt_tokens": 179315, "cached_input_tokens": 96268, "completion_tokens": 694, "thoughts_tokens": 1943, "total_tokens": 181952, "request_count": 8 } }, "agent_requests": 16, "main_agent_model_calls": 8, "tool_calls": 13, "main_agent_timed_model_calls": 8 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [169.846, 104.129, 25.638], "agent_gross_standard_cost_usd": 1.12365269, "judge_gross_standard_cost_usd": 0.1191375, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 1339721, "agent_prompt_tokens": 1326349, "agent_cached_input_tokens": 905682, "agent_thinking_tokens": 6653, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 207232, "cached_input_tokens": 14208, "completion_tokens": 3874, "thoughts_tokens": 5678, "total_tokens": 216784, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 1083610, "cached_input_tokens": 884308, "completion_tokens": 2363, "thoughts_tokens": 0, "total_tokens": 1085973, "request_count": 33 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32504, "cached_input_tokens": 7166, "completion_tokens": 306, "thoughts_tokens": 0, "total_tokens": 32810, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 3003, "cached_input_tokens": 0, "completion_tokens": 176, "thoughts_tokens": 975, "total_tokens": 4154, "request_count": 4 } }, "agent_requests": 47, "main_agent_model_calls": 33, "tool_calls": 41, "main_agent_timed_model_calls": 33 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [132.621, 106.795, 29.813], "agent_gross_standard_cost_usd": 0.8363749, "judge_gross_standard_cost_usd": 0.1147815, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 642396, "agent_prompt_tokens": 623528, "agent_cached_input_tokens": 251196, "agent_thinking_tokens": 13282, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 203899, "cached_input_tokens": 0, "completion_tokens": 3722, "thoughts_tokens": 6227, "total_tokens": 213848, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 383481, "cached_input_tokens": 251196, "completion_tokens": 1436, "thoughts_tokens": 5921, "total_tokens": 390838, "request_count": 12 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32494, "cached_input_tokens": 0, "completion_tokens": 251, "thoughts_tokens": 0, "total_tokens": 32745, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 3654, "cached_input_tokens": 0, "completion_tokens": 177, "thoughts_tokens": 1134, "total_tokens": 4965, "request_count": 4 } }, "agent_requests": 26, "main_agent_model_calls": 12, "tool_calls": 21, "main_agent_timed_model_calls": 12 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [106.811, 80.073, 19.584], "agent_gross_standard_cost_usd": 0.266681885, "judge_gross_standard_cost_usd": 0.0760395, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 411821, "agent_prompt_tokens": 398416, "agent_cached_input_tokens": 115380, "agent_thinking_tokens": 8266, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 187916, "cached_input_tokens": 10137, "completion_tokens": 3630, "thoughts_tokens": 6015, "total_tokens": 197561, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 194403, "cached_input_tokens": 98092, "completion_tokens": 1365, "thoughts_tokens": 2251, "total_tokens": 198019, "request_count": 12 }, "gemini-3.1-flash-lite": { "prompt_tokens": 16097, "cached_input_tokens": 7151, "completion_tokens": 144, "thoughts_tokens": 0, "total_tokens": 16241, "request_count": 2 } }, "agent_requests": 20, "main_agent_model_calls": 8, "tool_calls": 11, "main_agent_timed_model_calls": 8 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [121.257, 112.997, 70.263], "agent_gross_standard_cost_usd": 0.3968441, "judge_gross_standard_cost_usd": 0.095754, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 548260, "agent_prompt_tokens": 525660, "agent_cached_input_tokens": 168816, "agent_thinking_tokens": 16168, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 212219, "cached_input_tokens": 0, "completion_tokens": 4636, "thoughts_tokens": 5801, "total_tokens": 222656, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 280895, "cached_input_tokens": 154496, "completion_tokens": 1492, "thoughts_tokens": 10367, "total_tokens": 292754, "request_count": 14 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32546, "cached_input_tokens": 14320, "completion_tokens": 304, "thoughts_tokens": 0, "total_tokens": 32850, "request_count": 4 } }, "agent_requests": 25, "main_agent_model_calls": 10, "tool_calls": 16, "main_agent_timed_model_calls": 10 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [116.277, 91.521, 21.672], "agent_gross_standard_cost_usd": 0.35730269, "judge_gross_standard_cost_usd": 0.1352715, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 617492, "agent_prompt_tokens": 603944, "agent_cached_input_tokens": 243706, "agent_thinking_tokens": 7765, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 216805, "cached_input_tokens": 10138, "completion_tokens": 4002, "thoughts_tokens": 5166, "total_tokens": 225973, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 348606, "cached_input_tokens": 219254, "completion_tokens": 1316, "thoughts_tokens": 1400, "total_tokens": 351322, "request_count": 11 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32426, "cached_input_tokens": 14314, "completion_tokens": 289, "thoughts_tokens": 0, "total_tokens": 32715, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 6107, "cached_input_tokens": 0, "completion_tokens": 176, "thoughts_tokens": 1199, "total_tokens": 7482, "request_count": 4 } }, "agent_requests": 25, "main_agent_model_calls": 11, "tool_calls": 20, "main_agent_timed_model_calls": 11 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [89.523, 86.125, 18.814], "agent_gross_standard_cost_usd": 0.246385265, "judge_gross_standard_cost_usd": 0.077115, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 403441, "agent_prompt_tokens": 390901, "agent_cached_input_tokens": 121406, "agent_thinking_tokens": 7191, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 184690, "cached_input_tokens": 16328, "completion_tokens": 3758, "thoughts_tokens": 4754, "total_tokens": 193202, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 176014, "cached_input_tokens": 97927, "completion_tokens": 1166, "thoughts_tokens": 1474, "total_tokens": 178654, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 24073, "cached_input_tokens": 7151, "completion_tokens": 253, "thoughts_tokens": 0, "total_tokens": 24326, "request_count": 3 }, "gemini-3.8-flash": { "prompt_tokens": 6124, "cached_input_tokens": 0, "completion_tokens": 172, "thoughts_tokens": 963, "total_tokens": 7259, "request_count": 4 } }, "agent_requests": 21, "main_agent_model_calls": 8, "tool_calls": 12, "main_agent_timed_model_calls": 8 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [192.15, 118.0, 28.192], "agent_gross_standard_cost_usd": 0.304043535, "judge_gross_standard_cost_usd": 0.082584, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 458761, "agent_prompt_tokens": 446359, "agent_cached_input_tokens": 134660, "agent_thinking_tokens": 7121, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 185424, "cached_input_tokens": 10137, "completion_tokens": 3626, "thoughts_tokens": 4921, "total_tokens": 193971, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 222388, "cached_input_tokens": 110208, "completion_tokens": 1196, "thoughts_tokens": 1236, "total_tokens": 224820, "request_count": 10 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32430, "cached_input_tokens": 14315, "completion_tokens": 273, "thoughts_tokens": 0, "total_tokens": 32703, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 6117, "cached_input_tokens": 0, "completion_tokens": 186, "thoughts_tokens": 964, "total_tokens": 7267, "request_count": 4 } }, "agent_requests": 24, "main_agent_model_calls": 10, "tool_calls": 14, "main_agent_timed_model_calls": 10 }, { "task": "task-33", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [160.391, 94.394, 42.629], "agent_gross_standard_cost_usd": 0.3702626, "judge_gross_standard_cost_usd": 0.089808, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 461983, "agent_prompt_tokens": 442203, "agent_cached_input_tokens": 116916, "agent_thinking_tokens": 13711, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 187619, "cached_input_tokens": 0, "completion_tokens": 4173, "thoughts_tokens": 4929, "total_tokens": 196721, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 215531, "cached_input_tokens": 102592, "completion_tokens": 1384, "thoughts_tokens": 7714, "total_tokens": 224629, "request_count": 9 }, "gemini-3.1-flash-lite": { "prompt_tokens": 32600, "cached_input_tokens": 14324, "completion_tokens": 319, "thoughts_tokens": 0, "total_tokens": 32919, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 6453, "cached_input_tokens": 0, "completion_tokens": 193, "thoughts_tokens": 1068, "total_tokens": 7714, "request_count": 4 } }, "agent_requests": 23, "main_agent_model_calls": 9, "tool_calls": 14, "main_agent_timed_model_calls": 9 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [51.297, 59.285], "agent_gross_standard_cost_usd": 0.7222726, "judge_gross_standard_cost_usd": 0.0665655, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 449874, "agent_prompt_tokens": 435871, "agent_cached_input_tokens": 112441, "agent_thinking_tokens": 10211, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 196988, "cached_input_tokens": 0, "completion_tokens": 2787, "thoughts_tokens": 4859, "total_tokens": 204634, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 236493, "cached_input_tokens": 112441, "completion_tokens": 906, "thoughts_tokens": 5223, "total_tokens": 242622, "request_count": 8 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 39, "thoughts_tokens": 129, "total_tokens": 838, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1720, "cached_input_tokens": 0, "completion_tokens": 60, "thoughts_tokens": 0, "total_tokens": 1780, "request_count": 1 } }, "agent_requests": 15, "main_agent_model_calls": 8, "tool_calls": 8, "main_agent_timed_model_calls": 8 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [26.609, 30.263], "agent_gross_standard_cost_usd": 0.21467, "judge_gross_standard_cost_usd": 0.052233, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 345890, "agent_prompt_tokens": 337310, "agent_cached_input_tokens": 63463, "agent_thinking_tokens": 5231, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 190981, "cached_input_tokens": 0, "completion_tokens": 2633, "thoughts_tokens": 5060, "total_tokens": 198674, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 144750, "cached_input_tokens": 63463, "completion_tokens": 657, "thoughts_tokens": 171, "total_tokens": 145578, "request_count": 7 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1579, "cached_input_tokens": 0, "completion_tokens": 59, "thoughts_tokens": 0, "total_tokens": 1638, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 5, "main_agent_timed_model_calls": 6 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [58.022, 78.51], "agent_gross_standard_cost_usd": 0.33595055, "judge_gross_standard_cost_usd": 0.058908, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 391333, "agent_prompt_tokens": 372795, "agent_cached_input_tokens": 64349, "agent_thinking_tokens": 14698, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 195429, "cached_input_tokens": 0, "completion_tokens": 3048, "thoughts_tokens": 4651, "total_tokens": 203128, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 175718, "cached_input_tokens": 64349, "completion_tokens": 736, "thoughts_tokens": 10047, "total_tokens": 186501, "request_count": 8 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1648, "cached_input_tokens": 0, "completion_tokens": 56, "thoughts_tokens": 0, "total_tokens": 1704, "request_count": 1 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 6, "main_agent_timed_model_calls": 7 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [30.229, 33.296], "agent_gross_standard_cost_usd": 0.22996312, "judge_gross_standard_cost_usd": 0.057501, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 347131, "agent_prompt_tokens": 337106, "agent_cached_input_tokens": 68456, "agent_thinking_tokens": 6638, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 190610, "cached_input_tokens": 10134, "completion_tokens": 2649, "thoughts_tokens": 5021, "total_tokens": 198280, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 144240, "cached_input_tokens": 58322, "completion_tokens": 642, "thoughts_tokens": 1491, "total_tokens": 146373, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 38, "thoughts_tokens": 126, "total_tokens": 834, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1586, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 1644, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 5, "main_agent_timed_model_calls": 6 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.728, 36.983], "agent_gross_standard_cost_usd": 0.18917852, "judge_gross_standard_cost_usd": 0.0391725, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 292490, "agent_prompt_tokens": 281673, "agent_cached_input_tokens": 70289, "agent_thinking_tokens": 7499, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 172046, "cached_input_tokens": 26464, "completion_tokens": 2634, "thoughts_tokens": 5238, "total_tokens": 179918, "request_count": 5 }, "gemini-3.7-flash": { "prompt_tokens": 107368, "cached_input_tokens": 43825, "completion_tokens": 590, "thoughts_tokens": 2117, "total_tokens": 110075, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 36, "thoughts_tokens": 144, "total_tokens": 850, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1589, "cached_input_tokens": 0, "completion_tokens": 58, "thoughts_tokens": 0, "total_tokens": 1647, "request_count": 1 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [197.676, 128.49], "agent_gross_standard_cost_usd": 0.23629628, "judge_gross_standard_cost_usd": 0.042756, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 294784, "agent_prompt_tokens": 283023, "agent_cached_input_tokens": 26632, "agent_thinking_tokens": 8266, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 172697, "cached_input_tokens": 10136, "completion_tokens": 2724, "thoughts_tokens": 6377, "total_tokens": 181798, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 107994, "cached_input_tokens": 16496, "completion_tokens": 667, "thoughts_tokens": 1822, "total_tokens": 110483, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 45, "thoughts_tokens": 67, "total_tokens": 782, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 1662, "cached_input_tokens": 0, "completion_tokens": 59, "thoughts_tokens": 0, "total_tokens": 1721, "request_count": 1 } }, "agent_requests": 12, "main_agent_model_calls": 5, "tool_calls": 4, "main_agent_timed_model_calls": 5 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [58.516, 77.567], "agent_gross_standard_cost_usd": 0.21911898, "judge_gross_standard_cost_usd": 0.0395895, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 292669, "agent_prompt_tokens": 280993, "agent_cached_input_tokens": 39569, "agent_thinking_tokens": 8236, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 165975, "cached_input_tokens": 10136, "completion_tokens": 2748, "thoughts_tokens": 3650, "total_tokens": 172373, "request_count": 5 }, "gemini-3.5-flash": { "prompt_tokens": 96485, "cached_input_tokens": 29433, "completion_tokens": 443, "thoughts_tokens": 4418, "total_tokens": 101346, "request_count": 4 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 37, "thoughts_tokens": 168, "total_tokens": 875, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 17863, "cached_input_tokens": 0, "completion_tokens": 212, "thoughts_tokens": 0, "total_tokens": 18075, "request_count": 2 } }, "agent_requests": 12, "main_agent_model_calls": 4, "tool_calls": 4, "main_agent_timed_model_calls": 4 }, { "task": "task-34", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [29.48, 25.715], "agent_gross_standard_cost_usd": 0.39823749999999997, "judge_gross_standard_cost_usd": 0.0653085, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 372290, "agent_prompt_tokens": 365251, "agent_cached_input_tokens": 76171, "agent_thinking_tokens": 3842, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 184966, "cached_input_tokens": 0, "completion_tokens": 2440, "thoughts_tokens": 3709, "total_tokens": 191115, "request_count": 5 }, "gemini-4-argon": { "prompt_tokens": 149119, "cached_input_tokens": 76171, "completion_tokens": 502, "thoughts_tokens": 0, "total_tokens": 149621, "request_count": 5 }, "gemini-3.8-flash": { "prompt_tokens": 670, "cached_input_tokens": 0, "completion_tokens": 50, "thoughts_tokens": 133, "total_tokens": 853, "request_count": 1 }, "gemini-3.1-flash-lite": { "prompt_tokens": 30496, "cached_input_tokens": 0, "completion_tokens": 205, "thoughts_tokens": 0, "total_tokens": 30701, "request_count": 2 } }, "agent_requests": 13, "main_agent_model_calls": 5, "tool_calls": 7, "main_agent_timed_model_calls": 5 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [42.812, 19.675, 23.307], "agent_gross_standard_cost_usd": 0.20180473, "judge_gross_standard_cost_usd": 0.0406755, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 324986, "agent_prompt_tokens": 315106, "agent_cached_input_tokens": 64085, "agent_thinking_tokens": 5662, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 190422, "cached_input_tokens": 10136, "completion_tokens": 3463, "thoughts_tokens": 4713, "total_tokens": 198598, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 124684, "cached_input_tokens": 53949, "completion_tokens": 755, "thoughts_tokens": 949, "total_tokens": 126388, "request_count": 7 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 4, "main_agent_timed_model_calls": 6 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [60.057, 39.385, 54.009], "agent_gross_standard_cost_usd": 0.30917825, "judge_gross_standard_cost_usd": 0.0604995, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 381198, "agent_prompt_tokens": 359576, "agent_cached_input_tokens": 86361, "agent_thinking_tokens": 17421, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 200466, "cached_input_tokens": 10135, "completion_tokens": 3446, "thoughts_tokens": 5784, "total_tokens": 209696, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 159110, "cached_input_tokens": 76226, "completion_tokens": 755, "thoughts_tokens": 11637, "total_tokens": 171502, "request_count": 8 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 6, "main_agent_timed_model_calls": 7 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [35.727, 14.898, 22.269], "agent_gross_standard_cost_usd": 0.23416447999999998, "judge_gross_standard_cost_usd": 0.0502215, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 371488, "agent_prompt_tokens": 361461, "agent_cached_input_tokens": 88780, "agent_thinking_tokens": 5871, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 199741, "cached_input_tokens": 20271, "completion_tokens": 3398, "thoughts_tokens": 5718, "total_tokens": 208857, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 161043, "cached_input_tokens": 68509, "completion_tokens": 715, "thoughts_tokens": 0, "total_tokens": 161758, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 43, "thoughts_tokens": 153, "total_tokens": 873, "request_count": 1 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [49.671, 15.752, 19.694], "agent_gross_standard_cost_usd": 0.21563995, "judge_gross_standard_cost_usd": 0.042483, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 327545, "agent_prompt_tokens": 316384, "agent_cached_input_tokens": 54033, "agent_thinking_tokens": 6966, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 190720, "cached_input_tokens": 0, "completion_tokens": 3438, "thoughts_tokens": 4729, "total_tokens": 198887, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 124987, "cached_input_tokens": 54033, "completion_tokens": 714, "thoughts_tokens": 2136, "total_tokens": 127837, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 43, "thoughts_tokens": 101, "total_tokens": 821, "request_count": 1 } }, "agent_requests": 13, "main_agent_model_calls": 6, "tool_calls": 4, "main_agent_timed_model_calls": 6 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [137.901, 37.527, 69.854], "agent_gross_standard_cost_usd": 0.27524638, "judge_gross_standard_cost_usd": 0.0525075, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 368643, "agent_prompt_tokens": 355106, "agent_cached_input_tokens": 58784, "agent_thinking_tokens": 9024, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 200953, "cached_input_tokens": 10136, "completion_tokens": 3565, "thoughts_tokens": 7128, "total_tokens": 211646, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 153476, "cached_input_tokens": 48648, "completion_tokens": 900, "thoughts_tokens": 1820, "total_tokens": 156196, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 48, "thoughts_tokens": 76, "total_tokens": 801, "request_count": 1 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [127.017, 68.195, 68.221], "agent_gross_standard_cost_usd": 0.35254008, "judge_gross_standard_cost_usd": 0.057981, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 381814, "agent_prompt_tokens": 361858, "agent_cached_input_tokens": 53502, "agent_thinking_tokens": 15198, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 201398, "cached_input_tokens": 10136, "completion_tokens": 3806, "thoughts_tokens": 6451, "total_tokens": 211655, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 159783, "cached_input_tokens": 43366, "completion_tokens": 914, "thoughts_tokens": 8637, "total_tokens": 169334, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 38, "thoughts_tokens": 110, "total_tokens": 825, "request_count": 1 } }, "agent_requests": 14, "main_agent_model_calls": 7, "tool_calls": 6, "main_agent_timed_model_calls": 7 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 2.0, "customer_turn_seconds": [35.669, 30.117, 31.856], "agent_gross_standard_cost_usd": 0.75439167, "judge_gross_standard_cost_usd": 0.0956355, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 538074, "agent_prompt_tokens": 524252, "agent_cached_input_tokens": 164496, "agent_thinking_tokens": 8762, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 218981, "cached_input_tokens": 10139, "completion_tokens": 3694, "thoughts_tokens": 8639, "total_tokens": 231314, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 304594, "cached_input_tokens": 154357, "completion_tokens": 1328, "thoughts_tokens": 0, "total_tokens": 305922, "request_count": 10 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 38, "thoughts_tokens": 123, "total_tokens": 838, "request_count": 1 } }, "agent_requests": 17, "main_agent_model_calls": 10, "tool_calls": 13, "main_agent_timed_model_calls": 10 }, { "task": "task-35", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [125.494, 38.505, 32.311], "agent_gross_standard_cost_usd": 0.82834005, "judge_gross_standard_cost_usd": 0.092091, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 508247, "agent_prompt_tokens": 491660, "agent_cached_input_tokens": 140778, "agent_thinking_tokens": 11966, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 218923, "cached_input_tokens": 10135, "completion_tokens": 3595, "thoughts_tokens": 5393, "total_tokens": 227911, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 272060, "cached_input_tokens": 130643, "completion_tokens": 985, "thoughts_tokens": 6465, "total_tokens": 279510, "request_count": 9 }, "gemini-3.8-flash": { "prompt_tokens": 677, "cached_input_tokens": 0, "completion_tokens": 41, "thoughts_tokens": 108, "total_tokens": 826, "request_count": 1 } }, "agent_requests": 16, "main_agent_model_calls": 9, "tool_calls": 10, "main_agent_timed_model_calls": 9 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "baseline_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [100.595, 52.598, 75.64], "agent_gross_standard_cost_usd": 0.38410979, "judge_gross_standard_cost_usd": 0.052845, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 415753, "agent_prompt_tokens": 388400, "agent_cached_input_tokens": 93941, "agent_thinking_tokens": 22811, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 191255, "cached_input_tokens": 10133, "completion_tokens": 3537, "thoughts_tokens": 8114, "total_tokens": 202906, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 197145, "cached_input_tokens": 83808, "completion_tokens": 1005, "thoughts_tokens": 14697, "total_tokens": 212847, "request_count": 11 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 7, "main_agent_timed_model_calls": 9 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash37_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [36.14, 33.585, 20.205], "agent_gross_standard_cost_usd": 0.2800146, "judge_gross_standard_cost_usd": 0.0736965, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 451035, "agent_prompt_tokens": 439004, "agent_cached_input_tokens": 125498, "agent_thinking_tokens": 7495, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 210168, "cached_input_tokens": 0, "completion_tokens": 3489, "thoughts_tokens": 5931, "total_tokens": 219588, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 227419, "cached_input_tokens": 125498, "completion_tokens": 953, "thoughts_tokens": 1011, "total_tokens": 229383, "request_count": 9 }, "gemini-3.8-flash": { "prompt_tokens": 1417, "cached_input_tokens": 0, "completion_tokens": 94, "thoughts_tokens": 553, "total_tokens": 2064, "request_count": 2 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 8, "main_agent_timed_model_calls": 9 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash37_medium", "thinking_level": "medium", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [48.776, 31.657, 21.802], "agent_gross_standard_cost_usd": 0.28099184, "judge_gross_standard_cost_usd": 0.067812, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 445385, "agent_prompt_tokens": 431896, "agent_cached_input_tokens": 131506, "agent_thinking_tokens": 9054, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 209821, "cached_input_tokens": 10133, "completion_tokens": 3514, "thoughts_tokens": 6415, "total_tokens": 219750, "request_count": 6 }, "gemini-3.7-flash": { "prompt_tokens": 220666, "cached_input_tokens": 121373, "completion_tokens": 823, "thoughts_tokens": 2240, "total_tokens": 223729, "request_count": 9 }, "gemini-3.8-flash": { "prompt_tokens": 1409, "cached_input_tokens": 0, "completion_tokens": 98, "thoughts_tokens": 399, "total_tokens": 1906, "request_count": 2 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 7, "main_agent_timed_model_calls": 9 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash35_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [89.126, 112.595, 52.391], "agent_gross_standard_cost_usd": 0.28695413999999997, "judge_gross_standard_cost_usd": 0.0475695, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 415689, "agent_prompt_tokens": 401317, "agent_cached_input_tokens": 120572, "agent_thinking_tokens": 9639, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 191443, "cached_input_tokens": 20268, "completion_tokens": 3706, "thoughts_tokens": 7529, "total_tokens": 202678, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 208440, "cached_input_tokens": 100304, "completion_tokens": 938, "thoughts_tokens": 1714, "total_tokens": 211092, "request_count": 10 }, "gemini-3.8-flash": { "prompt_tokens": 1434, "cached_input_tokens": 0, "completion_tokens": 89, "thoughts_tokens": 396, "total_tokens": 1919, "request_count": 2 } }, "agent_requests": 18, "main_agent_model_calls": 10, "tool_calls": 7, "main_agent_timed_model_calls": 10 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "flash35_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": false, "judge_score": 5.0, "customer_turn_seconds": [42.063, 93.226, 103.93], "agent_gross_standard_cost_usd": 0.30333353, "judge_gross_standard_cost_usd": 0.0474495, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 337817, "agent_prompt_tokens": 315287, "agent_cached_input_tokens": 55171, "agent_thinking_tokens": 18178, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 188130, "cached_input_tokens": 10136, "completion_tokens": 3286, "thoughts_tokens": 9289, "total_tokens": 200705, "request_count": 6 }, "gemini-3.5-flash": { "prompt_tokens": 125656, "cached_input_tokens": 45035, "completion_tokens": 976, "thoughts_tokens": 8086, "total_tokens": 134718, "request_count": 7 }, "gemini-3.8-flash": { "prompt_tokens": 1501, "cached_input_tokens": 0, "completion_tokens": 90, "thoughts_tokens": 803, "total_tokens": 2394, "request_count": 2 } }, "agent_requests": 15, "main_agent_model_calls": 7, "tool_calls": 5, "main_agent_timed_model_calls": 7 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "argon_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [37.522, 30.18, 22.27], "agent_gross_standard_cost_usd": 0.62976727, "judge_gross_standard_cost_usd": 0.0853245, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 490849, "agent_prompt_tokens": 479952, "agent_cached_input_tokens": 164452, "agent_thinking_tokens": 6102, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 210821, "cached_input_tokens": 20269, "completion_tokens": 3698, "thoughts_tokens": 5785, "total_tokens": 220304, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 267712, "cached_input_tokens": 144183, "completion_tokens": 1008, "thoughts_tokens": 0, "total_tokens": 268720, "request_count": 9 }, "gemini-3.8-flash": { "prompt_tokens": 1419, "cached_input_tokens": 0, "completion_tokens": 89, "thoughts_tokens": 317, "total_tokens": 1825, "request_count": 2 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 10, "main_agent_timed_model_calls": 9 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "argon_high", "thinking_level": "high", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [59.185, 41.787, 33.649], "agent_gross_standard_cost_usd": 0.83655485, "judge_gross_standard_cost_usd": 0.0826545, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 558399, "agent_prompt_tokens": 541270, "agent_cached_input_tokens": 189897, "agent_thinking_tokens": 12192, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 213013, "cached_input_tokens": 10135, "completion_tokens": 3810, "thoughts_tokens": 6888, "total_tokens": 223711, "request_count": 6 }, "gemini-4-argon": { "prompt_tokens": 326833, "cached_input_tokens": 179762, "completion_tokens": 1033, "thoughts_tokens": 4890, "total_tokens": 332756, "request_count": 11 }, "gemini-3.8-flash": { "prompt_tokens": 1424, "cached_input_tokens": 0, "completion_tokens": 94, "thoughts_tokens": 414, "total_tokens": 1932, "request_count": 2 } }, "agent_requests": 19, "main_agent_model_calls": 11, "tool_calls": 11, "main_agent_timed_model_calls": 11 }, { "task": "task-36", "suite": "Return policy and confirmation", "configuration": "baseline_low", "thinking_level": "low", "reused_completed_observation": false, "families": ["hallucination/policy", "returns/exchanges", "tool-use correctness"], "native_trace_omitted_helper_requests": 0, "passed": true, "judge_score": 5.0, "customer_turn_seconds": [32.064, 32.162, 22.249], "agent_gross_standard_cost_usd": 0.30742235, "judge_gross_standard_cost_usd": 0.0694695, "simulator_gross_standard_cost_usd": 0, "agent_total_tokens": 445947, "agent_prompt_tokens": 431286, "agent_cached_input_tokens": 97557, "agent_thinking_tokens": 9851, "agent_usage_by_model": { "gemini-3.5-flash-lite": { "prompt_tokens": 210331, "cached_input_tokens": 0, "completion_tokens": 3744, "thoughts_tokens": 9329, "total_tokens": 223404, "request_count": 6 }, "gemini-3.8-flash": { "prompt_tokens": 220955, "cached_input_tokens": 97557, "completion_tokens": 1066, "thoughts_tokens": 522, "total_tokens": 222543, "request_count": 11 } }, "agent_requests": 17, "main_agent_model_calls": 9, "tool_calls": 7, "main_agent_timed_model_calls": 9 } ], "main_call_latency_basis": "Available native main-call timing events. Three Argon High token-limit responses lack those timing events; their request counts, cost, tokens and customer-turn latency remain included." }