eval/results.json
| 1 | { |
| 2 | "model": "rl-agent", |
| 3 | "questions_evaluated": { |
| 4 | "eval_in": 23024, |
| 5 | "eval_zs": 2400 |
| 6 | }, |
| 7 | "by_task_family": { |
| 8 | "eval_in": { |
| 9 | "conversation outcomes": { |
| 10 | "n": 3600, |
| 11 | "accuracy": 0.4817, |
| 12 | "ece": 0.0193, |
| 13 | "nll": 0.6933 |
| 14 | }, |
| 15 | "email triage": { |
| 16 | "n": 2691, |
| 17 | "accuracy": 0.7321, |
| 18 | "ece": 0.0172, |
| 19 | "nll": 0.5954 |
| 20 | }, |
| 21 | "emotion and tone": { |
| 22 | "n": 1825, |
| 23 | "accuracy": 0.9058, |
| 24 | "ece": 0.0183, |
| 25 | "nll": 0.2382 |
| 26 | }, |
| 27 | "inference and fact checking": { |
| 28 | "n": 3022, |
| 29 | "accuracy": 0.8832, |
| 30 | "ece": 0.054, |
| 31 | "nll": 0.3404 |
| 32 | }, |
| 33 | "instruction-following tasks": { |
| 34 | "n": 600, |
| 35 | "accuracy": 0.8783, |
| 36 | "ece": 0.0465, |
| 37 | "nll": 0.3021 |
| 38 | }, |
| 39 | "intent and routing": { |
| 40 | "n": 1475, |
| 41 | "accuracy": 0.9912, |
| 42 | "ece": 0.0085, |
| 43 | "nll": 0.1811 |
| 44 | }, |
| 45 | "moderation and safety": { |
| 46 | "n": 2708, |
| 47 | "accuracy": 0.9671, |
| 48 | "ece": 0.0613, |
| 49 | "nll": 0.1527 |
| 50 | }, |
| 51 | "reading comprehension": { |
| 52 | "n": 770, |
| 53 | "accuracy": 0.8468, |
| 54 | "ece": 0.0833, |
| 55 | "nll": 0.4086 |
| 56 | }, |
| 57 | "response quality scoring": { |
| 58 | "n": 3146, |
| 59 | "accuracy": 0.5814, |
| 60 | "ece": 0.023, |
| 61 | "nll": 1.0087 |
| 62 | }, |
| 63 | "robustness checks": { |
| 64 | "n": 744, |
| 65 | "accuracy": 0.8508, |
| 66 | "ece": 0.1085, |
| 67 | "nll": 1.0576 |
| 68 | }, |
| 69 | "search relevance": { |
| 70 | "n": 733, |
| 71 | "accuracy": 0.6276, |
| 72 | "ece": 0.066, |
| 73 | "nll": 0.7281 |
| 74 | }, |
| 75 | "sentiment and rating": { |
| 76 | "n": 961, |
| 77 | "accuracy": 0.4422, |
| 78 | "ece": 0.4384, |
| 79 | "nll": 3.545 |
| 80 | }, |
| 81 | "topic classification": { |
| 82 | "n": 749, |
| 83 | "accuracy": 0.9386, |
| 84 | "ece": 0.0285, |
| 85 | "nll": 0.1957 |
| 86 | } |
| 87 | }, |
| 88 | "eval_zs": { |
| 89 | "emotion and tone": { |
| 90 | "n": 600, |
| 91 | "accuracy": 0.5833, |
| 92 | "ece": 0.3178, |
| 93 | "nll": 1.9761 |
| 94 | }, |
| 95 | "instruction-following tasks": { |
| 96 | "n": 600, |
| 97 | "accuracy": 0.8633, |
| 98 | "ece": 0.0455, |
| 99 | "nll": 0.3187 |
| 100 | }, |
| 101 | "moderation and safety": { |
| 102 | "n": 600, |
| 103 | "accuracy": 0.7967, |
| 104 | "ece": 0.1713, |
| 105 | "nll": 1.4151 |
| 106 | }, |
| 107 | "sentiment and rating": { |
| 108 | "n": 600, |
| 109 | "accuracy": 0.3617, |
| 110 | "ece": 0.2915, |
| 111 | "nll": 1.7985 |
| 112 | } |
| 113 | } |
| 114 | }, |
| 115 | "calibration_temperature": [ |
| 116 | 1.6369030475616455, |
| 117 | 1.2514300346374512, |
| 118 | 1.983399510383606 |
| 119 | ], |
| 120 | "latency_ms": { |
| 121 | "1_questions": { |
| 122 | "p50_ms": 38.4, |
| 123 | "p95_ms": 42.1 |
| 124 | }, |
| 125 | "10_questions": { |
| 126 | "p50_ms": 156.0, |
| 127 | "p95_ms": 158.4 |
| 128 | }, |
| 129 | "50_questions": { |
| 130 | "p50_ms": 721.4, |
| 131 | "p95_ms": 733.0 |
| 132 | } |
| 133 | }, |
| 134 | "act_policy": { |
| 135 | "eval_in": { |
| 136 | "automation_rate": 1.0, |
| 137 | "accuracy_when_acting": 0.8032331136738056, |
| 138 | "accuracy_when_escalating": null, |
| 139 | "accuracy_all": 0.8032331136738056 |
| 140 | } |
| 141 | } |
| 142 | } |