{
 "workload": "w10",
 "suffix": "_w10",
 "libraos_suffix": "_w10b",
 "repeats": null,
 "frameworks": {
  "libraos": {
   "repeats": [
    "1",
    "2",
    "3"
   ],
   "cases": 300,
   "success": 298,
   "rate": 0.9933,
   "fail_class": {
    "incorrect_action": 2
   },
   "by_kind": {
    "defect_refund": "59/60",
    "discount": "60/60",
    "late_plus_deal": "60/60",
    "refund_note": "60/60",
    "return_refund": "59/60"
   },
   "by_variant": {
    "action": "158/159",
    "fault": "90/90",
    "refusal_correct": "50/51"
   },
   "duplicate_refunds": 0,
   "lost_response_cases": 30,
   "handoffs_per_case": 2.4,
   "errors": 0,
   "model_calls_per_case": 11.05,
   "prompt_tokens_per_case": 12289,
   "cache_hit": 0.169,
   "completion_tokens_per_case": 1731,
   "cost_usd": 31.2,
   "cost_per_case": 0.104,
   "cost_per_1k_success": 104.69,
   "mean_latency_s": 52.1
  },
  "agno": {
   "repeats": [
    "1",
    "2",
    "3"
   ],
   "cases": 300,
   "success": 299,
   "rate": 0.9967,
   "fail_class": {
    "incorrect_action": 1
   },
   "by_kind": {
    "defect_refund": "59/60",
    "discount": "60/60",
    "late_plus_deal": "60/60",
    "refund_note": "60/60",
    "return_refund": "60/60"
   },
   "by_variant": {
    "action": "158/159",
    "fault": "90/90",
    "refusal_correct": "51/51"
   },
   "duplicate_refunds": 0,
   "lost_response_cases": 30,
   "handoffs_per_case": 2.62,
   "errors": 0,
   "model_calls_per_case": 11.51,
   "prompt_tokens_per_case": 13946,
   "cache_hit": 0.212,
   "completion_tokens_per_case": 1877,
   "cost_usd": 33.82,
   "cost_per_case": 0.1127,
   "cost_per_1k_success": 113.11,
   "mean_latency_s": 50.7
  },
  "langgraph": {
   "repeats": [
    "1",
    "2",
    "3"
   ],
   "cases": 300,
   "success": 296,
   "rate": 0.9867,
   "fail_class": {
    "incorrect_refusal": 2,
    "incorrect_action": 2
   },
   "by_kind": {
    "defect_refund": "58/60",
    "discount": "58/60",
    "late_plus_deal": "60/60",
    "refund_note": "60/60",
    "return_refund": "60/60"
   },
   "by_variant": {
    "action": "157/159",
    "fault": "88/90",
    "refusal_correct": "51/51"
   },
   "duplicate_refunds": 0,
   "lost_response_cases": 30,
   "handoffs_per_case": 2.46,
   "errors": 0,
   "model_calls_per_case": 10.99,
   "prompt_tokens_per_case": 11856,
   "cache_hit": 0.162,
   "completion_tokens_per_case": 1713,
   "cost_usd": 30.61,
   "cost_per_case": 0.102,
   "cost_per_1k_success": 103.42,
   "mean_latency_s": 48.4
  }
 },
 "paired": {
  "agno": {
   "cases": 100,
   "diff_pp": -0.3,
   "ci95_pp": [
    -1.3,
    0.7
   ],
   "within_2pp": true,
   "libraos_only_fail": [
    "c1008",
    "c1081"
   ],
   "comp_only_fail": [
    "c1016"
   ]
  },
  "langgraph": {
   "cases": 100,
   "diff_pp": 0.7,
   "ci95_pp": [
    -1.0,
    2.3
   ],
   "within_2pp": true,
   "libraos_only_fail": [
    "c1008",
    "c1081"
   ],
   "comp_only_fail": [
    "c1053",
    "c1079",
    "c1085",
    "c1098"
   ]
  }
 }
}