reports/vllm_decisions_9b.json
| 1 | { |
| 2 | "bundle": "exports/jev-judge-qwen35-9b-v0.8", |
| 3 | "split": "test_set_30k", |
| 4 | "n": 29955, |
| 5 | "load_s": 287.309130191803, |
| 6 | "decide_s": 80.14112758636475, |
| 7 | "rows_per_s": 373.7781199511917, |
| 8 | "prompt_tok_per_s": 39862.9030588227, |
| 9 | "metrics": { |
| 10 | "n": 29955, |
| 11 | "kl": 0.02108642364071223, |
| 12 | "js": 0.005440059594873707, |
| 13 | "n_defined": 27575, |
| 14 | "top1": 0.91793292760849, |
| 15 | "ece": 0.000674150627137558, |
| 16 | "mce": 0.054795737295745106, |
| 17 | "noul_brier_soft": 0.0015240436662106603, |
| 18 | "noul_brier_hard": 0.07438912617278573, |
| 19 | "noul_auroc": 0.9958085476266227, |
| 20 | "score_mae_expected": 0.10307906399610614, |
| 21 | "score_rps": 0.00847646691286537, |
| 22 | "choice_top1": 0.8982551693916321, |
| 23 | "choice_kl": 0.04230141424342395 |
| 24 | }, |
| 25 | "vs_hf": { |
| 26 | "max_abs_dp": 0.25778780511591254, |
| 27 | "mean_abs_dp": 0.0012982510510488284, |
| 28 | "p99_abs_dp": 0.00865467635599359, |
| 29 | "argmax_agreement": 0.9912201635787014 |
| 30 | }, |
| 31 | "mixed_call_s": 0.6256270408630371, |
| 32 | "mixed_decisions_match": 0.002922693355067174, |
| 33 | "mixed_generation_sample": [ |
| 34 | "Thinking Process:\n\n1. **Analyze the Request:**\n * Topic: Safety Stock.\n * Constraint: One sentence.\n * Goal: Define/explain what it is clearly a", |
| 35 | "Thinking Process:\n\n1. **Analyze the Request:**\n * Task: Write a Python one-liner.\n * Functionality: Reverse a string.\n * Constraint: Must be a s" |
| 36 | ] |
| 37 | } |