reports/realworld_27b.json
48.4 KB · 1541 lines · json Raw
1 {
2 "bench": {
3 "seconds": 110.02985763549805,
4 "decisions": 14400,
5 "rq1_acc_by_k": {
6 "2": 0.87625,
7 "4": 0.78375,
8 "8": 0.7675,
9 "16": 0.74
10 },
11 "rq1_acc_by_domain_k16": {
12 "clinc": 0.93,
13 "dbpedia": 0.885,
14 "goemotions": 0.415,
15 "mtop": 0.73
16 },
17 "rq3": {
18 "2": {
19 "near": 0.9725,
20 "far": 1.0
21 },
22 "4": {
23 "near": 0.9575,
24 "far": 0.9925
25 },
26 "8": {
27 "near": 0.91,
28 "far": 0.985
29 },
30 "16": {
31 "near": 0.9075,
32 "far": 0.9825
33 }
34 },
35 "rq2_flip_k16_item_any_change": 0.151875,
36 "rq2_flip_k16_per_decision_vs_first": 0.07375,
37 "rq2_flip_by_domain_tier": {
38 "clinc|far": 0.0025,
39 "clinc|near": 0.0425,
40 "fintopic|far": 0.02125,
41 "fintopic|near": 0.085,
42 "goemotions|far": 0.1425,
43 "goemotions|near": 0.22125,
44 "mtop|far": 0.00875,
45 "mtop|near": 0.06625
46 },
47 "jev": {
48 "rq1_acc_by_k": {
49 "2": 0.89,
50 "4": 0.8013,
51 "8": 0.7825,
52 "16": 0.7688
53 },
54 "rq1_acc_by_domain_k16": {
55 "clinc": 0.945,
56 "dbpedia": 0.9,
57 "goemotions": 0.47,
58 "mtop": 0.76
59 },
60 "rq3": {
61 "2": {
62 "near": 0.985,
63 "far": 1.0
64 },
65 "4": {
66 "near": 0.97,
67 "far": 0.99
68 },
69 "8": {
70 "near": 0.9325,
71 "far": 0.98
72 },
73 "16": {
74 "near": 0.9125,
75 "far": 0.985
76 }
77 },
78 "rq2_flip_k16": 0.07,
79 "rq2_flip_by_domain_tier_k16": {
80 "clinc|far": 0.02,
81 "clinc|near": 0.015,
82 "fintopic|far": 0.02,
83 "fintopic|near": 0.08,
84 "goemotions|far": 0.105,
85 "goemotions|near": 0.235,
86 "mtop|far": 0.005,
87 "mtop|near": 0.08
88 }
89 }
90 },
91 "cases": [
92 {
93 "suite": "B. HN front page",
94 "case": "F-Droid 2.0",
95 "question": "Which topic is this story mainly about?",
96 "answer": "open-source software (0.95)",
97 "expected": "open-source software",
98 "ok": true,
99 "probs": {
100 "open-source software": 0.946264922618866,
101 "programming languages & tools": 0.0025682502891868353,
102 "AI / machine learning": 4.214549335301854e-05,
103 "security & privacy": 0.050840239971876144,
104 "hardware & retro computing": 0.00017778707842808217,
105 "science & health": 3.9474849472753704e-05,
106 "automotive & energy": 1.030758267006604e-05,
107 "politics & government": 2.4278072032757336e-06,
108 "business & finance": 6.608678177144611e-06,
109 "culture & lifestyle": 4.78478214063216e-05
110 }
111 },
112 {
113 "suite": "B. HN front page",
114 "case": "F-Droid 2.0",
115 "question": "Is this story about artificial intelligence or machine learning?",
116 "answer": "P(true)=0.09",
117 "expected": "false",
118 "ok": true,
119 "probs": {
120 "false": 0.9120426177978516,
121 "true": 0.08795731514692307
122 }
123 },
124 {
125 "suite": "B. HN front page",
126 "case": "Dutch governments builds alternative for Microsoft based on NixOS",
127 "question": "Which topic is this story mainly about?",
128 "answer": "open-source software (0.82)",
129 "expected": "open-source software / politics & government",
130 "ok": true,
131 "probs": {
132 "open-source software": 0.8159499764442444,
133 "programming languages & tools": 0.017718730494379997,
134 "AI / machine learning": 0.0002013609919231385,
135 "security & privacy": 0.017991555854678154,
136 "hardware & retro computing": 0.0006808804464526474,
137 "science & health": 0.0005370366270653903,
138 "automotive & energy": 0.00020070167374797165,
139 "politics & government": 0.1462879329919815,
140 "business & finance": 0.00015078768774401397,
141 "culture & lifestyle": 0.00028099422343075275
142 }
143 },
144 {
145 "suite": "B. HN front page",
146 "case": "Dutch governments builds alternative for Microsoft based on NixOS",
147 "question": "Is this story about artificial intelligence or machine learning?",
148 "answer": "P(true)=0.04",
149 "expected": "false",
150 "ok": true,
151 "probs": {
152 "false": 0.9600890278816223,
153 "true": 0.03991095721721649
154 }
155 },
156 {
157 "suite": "B. HN front page",
158 "case": "Two-tier encryption in the UK",
159 "question": "Which topic is this story mainly about?",
160 "answer": "security & privacy (0.99)",
161 "expected": "politics & government / security & privacy",
162 "ok": true,
163 "probs": {
164 "open-source software": 6.620938802370802e-05,
165 "programming languages & tools": 0.00024466877221129835,
166 "AI / machine learning": 6.451978697441518e-05,
167 "security & privacy": 0.9918436408042908,
168 "hardware & retro computing": 9.669075370766222e-05,
169 "science & health": 4.84324773424305e-05,
170 "automotive & energy": 7.067960268614115e-06,
171 "politics & government": 0.007603170350193977,
172 "business & finance": 4.873612397204852e-06,
173 "culture & lifestyle": 2.079710247926414e-05
174 }
175 },
176 {
177 "suite": "B. HN front page",
178 "case": "Two-tier encryption in the UK",
179 "question": "Is this story about artificial intelligence or machine learning?",
180 "answer": "P(true)=0.09",
181 "expected": "false",
182 "ok": true,
183 "probs": {
184 "false": 0.9073770046234131,
185 "true": 0.09262295067310333
186 }
187 },
188 {
189 "suite": "B. HN front page",
190 "case": "Why is the liver so weirdly regenerative?",
191 "question": "Which topic is this story mainly about?",
192 "answer": "science & health (1.00)",
193 "expected": "science & health",
194 "ok": true,
195 "probs": {
196 "open-source software": 3.083374053858279e-07,
197 "programming languages & tools": 3.0879868972988334e-06,
198 "AI / machine learning": 3.5499997466104105e-06,
199 "security & privacy": 6.889530595799442e-06,
200 "hardware & retro computing": 1.4932202248019166e-06,
201 "science & health": 0.9999843835830688,
202 "automotive & energy": 1.90339708439069e-08,
203 "politics & government": 2.2335546745466672e-08,
204 "business & finance": 4.096373462658676e-09,
205 "culture & lifestyle": 2.2981730296578462e-07
206 }
207 },
208 {
209 "suite": "B. HN front page",
210 "case": "Why is the liver so weirdly regenerative?",
211 "question": "Is this story about artificial intelligence or machine learning?",
212 "answer": "P(true)=0.04",
213 "expected": "false",
214 "ok": true,
215 "probs": {
216 "false": 0.9604007601737976,
217 "true": 0.03959919512271881
218 }
219 },
220 {
221 "suite": "B. HN front page",
222 "case": "Toyota is taking the Corolla electric",
223 "question": "Which topic is this story mainly about?",
224 "answer": "automotive & energy (1.00)",
225 "expected": "automotive & energy",
226 "ok": true,
227 "probs": {
228 "open-source software": 1.9699729136846145e-07,
229 "programming languages & tools": 8.132251991810335e-07,
230 "AI / machine learning": 6.8380936681933235e-06,
231 "security & privacy": 8.479263669869397e-06,
232 "hardware & retro computing": 1.1175871804880444e-05,
233 "science & health": 5.970130132482154e-06,
234 "automotive & energy": 0.9999104738235474,
235 "politics & government": 8.149817176672514e-08,
236 "business & finance": 5.4012765758670866e-05,
237 "culture & lifestyle": 1.918673433465301e-06
238 }
239 },
240 {
241 "suite": "B. HN front page",
242 "case": "Toyota is taking the Corolla electric",
243 "question": "Is this story about artificial intelligence or machine learning?",
244 "answer": "P(true)=0.07",
245 "expected": "false",
246 "ok": true,
247 "probs": {
248 "false": 0.9311957359313965,
249 "true": 0.06880427896976471
250 }
251 },
252 {
253 "suite": "B. HN front page",
254 "case": "Rails World 2026 Opening Keynote [video]",
255 "question": "Which topic is this story mainly about?",
256 "answer": "programming languages & tools (0.92)",
257 "expected": "programming languages & tools",
258 "ok": true,
259 "probs": {
260 "open-source software": 0.07720935344696045,
261 "programming languages & tools": 0.9216884970664978,
262 "AI / machine learning": 0.0002499347901903093,
263 "security & privacy": 0.00014379659842234105,
264 "hardware & retro computing": 0.00010849555837921798,
265 "science & health": 0.00017547230527270585,
266 "automotive & energy": 1.2728226465696935e-05,
267 "politics & government": 4.617214472091291e-06,
268 "business & finance": 9.021761798067018e-05,
269 "culture & lifestyle": 0.0003168575349263847
270 }
271 },
272 {
273 "suite": "B. HN front page",
274 "case": "Rails World 2026 Opening Keynote [video]",
275 "question": "Is this story about artificial intelligence or machine learning?",
276 "answer": "P(true)=0.17",
277 "expected": "false",
278 "ok": true,
279 "probs": {
280 "false": 0.829389750957489,
281 "true": 0.170610249042511
282 }
283 },
284 {
285 "suite": "B. HN front page",
286 "case": "Opus 5.5 is good at explainer videos",
287 "question": "Which topic is this story mainly about?",
288 "answer": "AI / machine learning (0.99)",
289 "expected": "AI / machine learning",
290 "ok": true,
291 "probs": {
292 "open-source software": 0.00033625776995904744,
293 "programming languages & tools": 0.004068075213581324,
294 "AI / machine learning": 0.9937840700149536,
295 "security & privacy": 0.00014119860134087503,
296 "hardware & retro computing": 8.251640247181058e-05,
297 "science & health": 0.00013905421656090766,
298 "automotive & energy": 7.138335604395252e-06,
299 "politics & government": 2.963404540423653e-06,
300 "business & finance": 0.00010425073560327291,
301 "culture & lifestyle": 0.0013345073675736785
302 }
303 },
304 {
305 "suite": "B. HN front page",
306 "case": "Opus 5.5 is good at explainer videos",
307 "question": "Is this story about artificial intelligence or machine learning?",
308 "answer": "P(true)=0.85",
309 "expected": "true",
310 "ok": true,
311 "probs": {
312 "false": 0.15116219222545624,
313 "true": 0.8488377928733826
314 }
315 },
316 {
317 "suite": "B. HN front page",
318 "case": "Fearless SIMD v1.0",
319 "question": "Which topic is this story mainly about?",
320 "answer": "programming languages & tools (0.82)",
321 "expected": "programming languages & tools",
322 "ok": true,
323 "probs": {
324 "open-source software": 0.05537916719913483,
325 "programming languages & tools": 0.8172256350517273,
326 "AI / machine learning": 0.003944867756217718,
327 "security & privacy": 0.0035071310121566057,
328 "hardware & retro computing": 0.11869298666715622,
329 "science & health": 0.0007320266449823976,
330 "automotive & energy": 0.00013769918587058783,
331 "politics & government": 2.620266968733631e-05,
332 "business & finance": 3.0215860533644445e-05,
333 "culture & lifestyle": 0.0003240494115743786
334 }
335 },
336 {
337 "suite": "B. HN front page",
338 "case": "Fearless SIMD v1.0",
339 "question": "Is this story about artificial intelligence or machine learning?",
340 "answer": "P(true)=0.16",
341 "expected": "false",
342 "ok": true,
343 "probs": {
344 "false": 0.8388583660125732,
345 "true": 0.16114164888858795
346 }
347 },
348 {
349 "suite": "B. HN front page",
350 "case": "Pentium II at 600Mhz with Voodoo 3 Emulated on 86Box with M6 Mac Mini",
351 "question": "Which topic is this story mainly about?",
352 "answer": "hardware & retro computing (1.00)",
353 "expected": "hardware & retro computing",
354 "ok": true,
355 "probs": {
356 "open-source software": 0.000460192997707054,
357 "programming languages & tools": 3.850698340102099e-05,
358 "AI / machine learning": 2.0458423932723235e-06,
359 "security & privacy": 5.300806151353754e-06,
360 "hardware & retro computing": 0.9994930028915405,
361 "science & health": 3.587144306038681e-07,
362 "automotive & energy": 1.8996288986272702e-08,
363 "politics & government": 2.779793817353493e-08,
364 "business & finance": 1.5696747013294043e-08,
365 "culture & lifestyle": 6.254014692785859e-07
366 }
367 },
368 {
369 "suite": "B. HN front page",
370 "case": "Pentium II at 600Mhz with Voodoo 3 Emulated on 86Box with M6 Mac Mini",
371 "question": "Is this story about artificial intelligence or machine learning?",
372 "answer": "P(true)=0.03",
373 "expected": "false",
374 "ok": true,
375 "probs": {
376 "false": 0.9688998460769653,
377 "true": 0.03110017068684101
378 }
379 },
380 {
381 "suite": "B. HN front page",
382 "case": "The Mafia may be keeping fentanyl out of Italy",
383 "question": "Which topic is this story mainly about?",
384 "answer": "science & health (0.61)",
385 "expected": "politics & government / science & health",
386 "ok": true,
387 "probs": {
388 "open-source software": 0.0002378881472395733,
389 "programming languages & tools": 0.0013201786205172539,
390 "AI / machine learning": 0.0019494330044835806,
391 "security & privacy": 0.09407573938369751,
392 "hardware & retro computing": 0.0012619850458577275,
393 "science & health": 0.6130920648574829,
394 "automotive & energy": 0.0005107872420921922,
395 "politics & government": 0.10884310305118561,
396 "business & finance": 0.04391257464885712,
397 "culture & lifestyle": 0.13479627668857574
398 }
399 },
400 {
401 "suite": "B. HN front page",
402 "case": "The Mafia may be keeping fentanyl out of Italy",
403 "question": "Is this story about artificial intelligence or machine learning?",
404 "answer": "P(true)=0.02",
405 "expected": "false",
406 "ok": true,
407 "probs": {
408 "false": 0.9763078689575195,
409 "true": 0.02369212917983532
410 }
411 },
412 {
413 "suite": "B. HN front page",
414 "case": "Oracle on the hook to pay data centre investors even if site has no electricity",
415 "question": "Which topic is this story mainly about?",
416 "answer": "business & finance (1.00)",
417 "expected": "business & finance",
418 "ok": true,
419 "probs": {
420 "open-source software": 5.908738103244104e-07,
421 "programming languages & tools": 2.824800276357564e-06,
422 "AI / machine learning": 0.0005129554192535579,
423 "security & privacy": 2.0072646293556318e-05,
424 "hardware & retro computing": 3.7470468669198453e-05,
425 "science & health": 9.751320249051787e-06,
426 "automotive & energy": 0.0042397575452923775,
427 "politics & government": 1.0067228686239105e-06,
428 "business & finance": 0.9951735138893127,
429 "culture & lifestyle": 2.0730517462652642e-06
430 }
431 },
432 {
433 "suite": "B. HN front page",
434 "case": "Oracle on the hook to pay data centre investors even if site has no electricity",
435 "question": "Is this story about artificial intelligence or machine learning?",
436 "answer": "P(true)=0.15",
437 "expected": "false",
438 "ok": true,
439 "probs": {
440 "false": 0.8483027815818787,
441 "true": 0.15169720351696014
442 }
443 },
444 {
445 "suite": "B. HN front page",
446 "case": "CVE-2025-13032: Entering and Breaking the Avast Antivirus Sandbox Part 2",
447 "question": "Which topic is this story mainly about?",
448 "answer": "security & privacy (1.00)",
449 "expected": "security & privacy",
450 "ok": true,
451 "probs": {
452 "open-source software": 1.054316817317158e-05,
453 "programming languages & tools": 0.00010297320841345936,
454 "AI / machine learning": 2.652191824381589e-06,
455 "security & privacy": 0.9998821020126343,
456 "hardware & retro computing": 1.4209732626113691e-06,
457 "science & health": 2.339900504466641e-07,
458 "automotive & energy": 1.4718415819459096e-08,
459 "politics & government": 1.118017234347235e-08,
460 "business & finance": 2.687420508351579e-09,
461 "culture & lifestyle": 2.9283015123837686e-08
462 }
463 },
464 {
465 "suite": "B. HN front page",
466 "case": "CVE-2025-13032: Entering and Breaking the Avast Antivirus Sandbox Part 2",
467 "question": "Is this story about artificial intelligence or machine learning?",
468 "answer": "P(true)=0.08",
469 "expected": "false",
470 "ok": true,
471 "probs": {
472 "false": 0.918955385684967,
473 "true": 0.08104456216096878
474 }
475 },
476 {
477 "suite": "B. HN front page",
478 "case": "Platform-Independent SIMD in Go",
479 "question": "Which topic is this story mainly about?",
480 "answer": "programming languages & tools (0.99)",
481 "expected": "programming languages & tools",
482 "ok": true,
483 "probs": {
484 "open-source software": 0.000564372050575912,
485 "programming languages & tools": 0.9944759011268616,
486 "AI / machine learning": 3.779905091505498e-05,
487 "security & privacy": 1.0280105016136076e-05,
488 "hardware & retro computing": 0.004899586550891399,
489 "science & health": 1.0488091902516317e-05,
490 "automotive & energy": 3.322061843391566e-07,
491 "politics & government": 3.0862335620440717e-07,
492 "business & finance": 6.955661291385695e-08,
493 "culture & lifestyle": 9.757268344401382e-07
494 }
495 },
496 {
497 "suite": "B. HN front page",
498 "case": "Platform-Independent SIMD in Go",
499 "question": "Is this story about artificial intelligence or machine learning?",
500 "answer": "P(true)=0.09",
501 "expected": "false",
502 "ok": true,
503 "probs": {
504 "false": 0.9147270917892456,
505 "true": 0.08527292311191559
506 }
507 },
508 {
509 "suite": "B. HN front page",
510 "case": "Git-bug: Distributed, offline-first bug tracker embedded in Git",
511 "question": "Which topic is this story mainly about?",
512 "answer": "programming languages & tools (0.72)",
513 "expected": "open-source software / programming languages & tools",
514 "ok": true,
515 "probs": {
516 "open-source software": 0.27963703870773315,
517 "programming languages & tools": 0.7182855010032654,
518 "AI / machine learning": 4.650293703889474e-05,
519 "security & privacy": 0.0015319745289161801,
520 "hardware & retro computing": 0.00014946564624551684,
521 "science & health": 0.0002291313576279208,
522 "automotive & energy": 3.226467742933892e-05,
523 "politics & government": 8.985479325929191e-06,
524 "business & finance": 1.3740488611801993e-05,
525 "culture & lifestyle": 6.539292371599004e-05
526 }
527 },
528 {
529 "suite": "B. HN front page",
530 "case": "Git-bug: Distributed, offline-first bug tracker embedded in Git",
531 "question": "Is this story about artificial intelligence or machine learning?",
532 "answer": "P(true)=0.03",
533 "expected": "false",
534 "ok": true,
535 "probs": {
536 "false": 0.9667761325836182,
537 "true": 0.03322387859225273
538 }
539 },
540 {
541 "suite": "B. HN front page",
542 "case": "Using LLMs to trace alchemical knowledge and decode 17th century letters",
543 "question": "Which topic is this story mainly about?",
544 "answer": "AI / machine learning (0.96)",
545 "expected": "AI / machine learning / culture & lifestyle",
546 "ok": true,
547 "probs": {
548 "open-source software": 8.210006490116939e-05,
549 "programming languages & tools": 0.0012425229651853442,
550 "AI / machine learning": 0.9627402424812317,
551 "security & privacy": 6.486645725090057e-05,
552 "hardware & retro computing": 0.0001610809558769688,
553 "science & health": 0.016674496233463287,
554 "automotive & energy": 6.292930265772156e-06,
555 "politics & government": 6.341094831441296e-06,
556 "business & finance": 2.622830834297929e-06,
557 "culture & lifestyle": 0.019019372761249542
558 }
559 },
560 {
561 "suite": "B. HN front page",
562 "case": "Using LLMs to trace alchemical knowledge and decode 17th century letters",
563 "question": "Is this story about artificial intelligence or machine learning?",
564 "answer": "P(true)=0.92",
565 "expected": "true",
566 "ok": true,
567 "probs": {
568 "false": 0.07567980140447617,
569 "true": 0.9243202209472656
570 }
571 },
572 {
573 "suite": "B. HN front page",
574 "case": "Show HN: Agentic CUDA Kernel Optimizer",
575 "question": "Which topic is this story mainly about?",
576 "answer": "programming languages & tools (0.44)",
577 "expected": "AI / machine learning / programming languages & tools",
578 "ok": true,
579 "probs": {
580 "open-source software": 0.13229449093341827,
581 "programming languages & tools": 0.437865674495697,
582 "AI / machine learning": 0.38795411586761475,
583 "security & privacy": 0.00038756459252908826,
584 "hardware & retro computing": 0.03935153782367706,
585 "science & health": 0.0012956958962604403,
586 "automotive & energy": 0.0003036174748558551,
587 "politics & government": 6.544907228089869e-05,
588 "business & finance": 6.291853060247377e-05,
589 "culture & lifestyle": 0.0004189629398752004
590 }
591 },
592 {
593 "suite": "B. HN front page",
594 "case": "Show HN: Agentic CUDA Kernel Optimizer",
595 "question": "Is this story about artificial intelligence or machine learning?",
596 "answer": "P(true)=0.71",
597 "expected": "true",
598 "ok": true,
599 "probs": {
600 "false": 0.2896099090576172,
601 "true": 0.7103900909423828
602 }
603 },
604 {
605 "suite": "B. HN front page",
606 "case": "Amiga Screens: A Primer",
607 "question": "Which topic is this story mainly about?",
608 "answer": "hardware & retro computing (1.00)",
609 "expected": "hardware & retro computing",
610 "ok": true,
611 "probs": {
612 "open-source software": 9.880752622848377e-05,
613 "programming languages & tools": 0.0013057341566309333,
614 "AI / machine learning": 6.146925898065092e-06,
615 "security & privacy": 2.4441422283416614e-05,
616 "hardware & retro computing": 0.9984785914421082,
617 "science & health": 2.518933342798846e-06,
618 "automotive & energy": 2.2679763844735135e-07,
619 "politics & government": 1.7832118714977696e-07,
620 "business & finance": 1.5237382910981978e-07,
621 "culture & lifestyle": 8.325380622409284e-05
622 }
623 },
624 {
625 "suite": "B. HN front page",
626 "case": "Amiga Screens: A Primer",
627 "question": "Is this story about artificial intelligence or machine learning?",
628 "answer": "P(true)=0.05",
629 "expected": "false",
630 "ok": true,
631 "probs": {
632 "false": 0.9543537497520447,
633 "true": 0.04564627632498741
634 }
635 },
636 {
637 "suite": "B. HN front page",
638 "case": "My weird new hobby: Wandering around Tokyo on Google Maps",
639 "question": "Which topic is this story mainly about?",
640 "answer": "culture & lifestyle (0.99)",
641 "expected": "culture & lifestyle",
642 "ok": true,
643 "probs": {
644 "open-source software": 0.00018791062757372856,
645 "programming languages & tools": 0.009483021683990955,
646 "AI / machine learning": 0.002193150809034705,
647 "security & privacy": 0.0021431036293506622,
648 "hardware & retro computing": 0.00019856197468470782,
649 "science & health": 0.00015379073738586158,
650 "automotive & energy": 1.6763007806730457e-05,
651 "politics & government": 3.6141082091489807e-06,
652 "business & finance": 4.7114658627833705e-06,
653 "culture & lifestyle": 0.9856153726577759
654 }
655 },
656 {
657 "suite": "B. HN front page",
658 "case": "My weird new hobby: Wandering around Tokyo on Google Maps",
659 "question": "Is this story about artificial intelligence or machine learning?",
660 "answer": "P(true)=0.07",
661 "expected": "false",
662 "ok": true,
663 "probs": {
664 "false": 0.9272943735122681,
665 "true": 0.07270567864179611
666 }
667 },
668 {
669 "suite": "B. HN front page",
670 "case": "Topcoat is pushing the boundary of server applications with Rust",
671 "question": "Which topic is this story mainly about?",
672 "answer": "programming languages & tools (0.97)",
673 "expected": "open-source software / programming languages & tools",
674 "ok": true,
675 "probs": {
676 "open-source software": 0.028008051216602325,
677 "programming languages & tools": 0.9714831113815308,
678 "AI / machine learning": 2.961634709208738e-05,
679 "security & privacy": 0.00032153347274288535,
680 "hardware & retro computing": 4.3332649511285126e-05,
681 "science & health": 7.946323603391647e-05,
682 "automotive & energy": 8.485192665830255e-06,
683 "politics & government": 2.289076974193449e-06,
684 "business & finance": 1.3127480997354724e-05,
685 "culture & lifestyle": 1.1050177818106022e-05
686 }
687 },
688 {
689 "suite": "B. HN front page",
690 "case": "Topcoat is pushing the boundary of server applications with Rust",
691 "question": "Is this story about artificial intelligence or machine learning?",
692 "answer": "P(true)=0.06",
693 "expected": "false",
694 "ok": true,
695 "probs": {
696 "false": 0.9407718777656555,
697 "true": 0.0592280812561512
698 }
699 },
700 {
701 "suite": "C. HN comments",
702 "case": ">...only Signal completely opts out of iCloud... There is an iCloud op",
703 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
704 "answer": "P(true)=0.01",
705 "expected": "false",
706 "ok": true,
707 "probs": {
708 "false": 0.9877684712409973,
709 "true": 0.012231521308422089
710 }
711 },
712 {
713 "suite": "C. HN comments",
714 "case": ">...only Signal completely opts out of iCloud... There is an iCloud op",
715 "question": "What is this comment mainly doing?",
716 "answer": "providing factual information (0.91)",
717 "expected": "providing factual information",
718 "ok": true,
719 "probs": {
720 "asking a question": 0.06017313897609711,
721 "sharing personal experience": 0.0010152851464226842,
722 "providing factual information": 0.9108976721763611,
723 "arguing an opinion": 0.011393971741199493,
724 "attacking another commenter": 0.014930167235434055,
725 "moderating the discussion": 0.0015897362027317286
726 }
727 },
728 {
729 "suite": "C. HN comments",
730 "case": "You're not British. Probably, you've never been to the UK. Why are you",
731 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
732 "answer": "P(true)=0.49",
733 "expected": "true",
734 "ok": false,
735 "probs": {
736 "false": 0.5115004181861877,
737 "true": 0.4884995222091675
738 }
739 },
740 {
741 "suite": "C. HN comments",
742 "case": "You're not British. Probably, you've never been to the UK. Why are you",
743 "question": "What is this comment mainly doing?",
744 "answer": "attacking another commenter (0.52)",
745 "expected": "arguing an opinion / attacking another commenter",
746 "ok": true,
747 "probs": {
748 "asking a question": 0.013749495148658752,
749 "sharing personal experience": 0.11412913352251053,
750 "providing factual information": 0.0032820755150169134,
751 "arguing an opinion": 0.3436119556427002,
752 "attacking another commenter": 0.52373206615448,
753 "moderating the discussion": 0.0014953016070649028
754 }
755 },
756 {
757 "suite": "C. HN comments",
758 "case": "> WhatsApp is similar by the way. Unless you enable E2E backups, they ",
759 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
760 "answer": "P(true)=0.01",
761 "expected": "false",
762 "ok": true,
763 "probs": {
764 "false": 0.9893696308135986,
765 "true": 0.010630309581756592
766 }
767 },
768 {
769 "suite": "C. HN comments",
770 "case": "> WhatsApp is similar by the way. Unless you enable E2E backups, they ",
771 "question": "What is this comment mainly doing?",
772 "answer": "providing factual information (0.65)",
773 "expected": "providing factual information",
774 "ok": true,
775 "probs": {
776 "asking a question": 0.0018196083838120103,
777 "sharing personal experience": 0.0014435279881581664,
778 "providing factual information": 0.6503943800926208,
779 "arguing an opinion": 0.3452976644039154,
780 "attacking another commenter": 0.000447586877271533,
781 "moderating the discussion": 0.0005971682257950306
782 }
783 },
784 {
785 "suite": "C. HN comments",
786 "case": "The article mentions that Cellebrite managed to gain access to the pho",
787 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
788 "answer": "P(true)=0.05",
789 "expected": "false",
790 "ok": true,
791 "probs": {
792 "false": 0.9494494199752808,
793 "true": 0.05055060610175133
794 }
795 },
796 {
797 "suite": "C. HN comments",
798 "case": "The article mentions that Cellebrite managed to gain access to the pho",
799 "question": "What is this comment mainly doing?",
800 "answer": "asking a question (0.99)",
801 "expected": "asking a question",
802 "ok": true,
803 "probs": {
804 "asking a question": 0.9850285053253174,
805 "sharing personal experience": 4.030948730360251e-06,
806 "providing factual information": 7.669319893466309e-05,
807 "arguing an opinion": 0.014881090261042118,
808 "attacking another commenter": 4.360715138318483e-06,
809 "moderating the discussion": 5.336835329217138e-06
810 }
811 },
812 {
813 "suite": "C. HN comments",
814 "case": "For anyone reading, this is entirely false. As sdcfgy says, these even",
815 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
816 "answer": "P(true)=0.04",
817 "expected": "false",
818 "ok": true,
819 "probs": {
820 "false": 0.9638738036155701,
821 "true": 0.03612624481320381
822 }
823 },
824 {
825 "suite": "C. HN comments",
826 "case": "For anyone reading, this is entirely false. As sdcfgy says, these even",
827 "question": "What is this comment mainly doing?",
828 "answer": "providing factual information (0.65)",
829 "expected": "arguing an opinion / providing factual information",
830 "ok": true,
831 "probs": {
832 "asking a question": 0.0002463263808749616,
833 "sharing personal experience": 0.0019767857156693935,
834 "providing factual information": 0.65326327085495,
835 "arguing an opinion": 0.211765319108963,
836 "attacking another commenter": 0.08069321513175964,
837 "moderating the discussion": 0.052055101841688156
838 }
839 },
840 {
841 "suite": "C. HN comments",
842 "case": "I moved my Apple account region to Spain and used a Banco Santander ba",
843 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
844 "answer": "P(true)=0.01",
845 "expected": "false",
846 "ok": true,
847 "probs": {
848 "false": 0.9900151491165161,
849 "true": 0.009984900243580341
850 }
851 },
852 {
853 "suite": "C. HN comments",
854 "case": "I moved my Apple account region to Spain and used a Banco Santander ba",
855 "question": "What is this comment mainly doing?",
856 "answer": "sharing personal experience (0.90)",
857 "expected": "sharing personal experience",
858 "ok": true,
859 "probs": {
860 "asking a question": 0.0001567534200148657,
861 "sharing personal experience": 0.9009243845939636,
862 "providing factual information": 0.0908980742096901,
863 "arguing an opinion": 0.007215847261250019,
864 "attacking another commenter": 0.00022715968952979892,
865 "moderating the discussion": 0.0005777014885097742
866 }
867 },
868 {
869 "suite": "C. HN comments",
870 "case": "How will you insert your physical ID card into your computer to prove ",
871 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
872 "answer": "P(true)=0.16",
873 "expected": "false",
874 "ok": true,
875 "probs": {
876 "false": 0.8441463708877563,
877 "true": 0.15585365891456604
878 }
879 },
880 {
881 "suite": "C. HN comments",
882 "case": "How will you insert your physical ID card into your computer to prove ",
883 "question": "What is this comment mainly doing?",
884 "answer": "arguing an opinion (0.69)",
885 "expected": "arguing an opinion / asking a question",
886 "ok": true,
887 "probs": {
888 "asking a question": 0.2930069863796234,
889 "sharing personal experience": 0.0005003541009500623,
890 "providing factual information": 0.0022376603446900845,
891 "arguing an opinion": 0.6937043070793152,
892 "attacking another commenter": 0.010148239322006702,
893 "moderating the discussion": 0.000402397767174989
894 }
895 },
896 {
897 "suite": "C. HN comments",
898 "case": ">There's a huge spectrum of, let's call it, aggressive speech, ranging",
899 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
900 "answer": "P(true)=0.69",
901 "expected": "true",
902 "ok": true,
903 "probs": {
904 "false": 0.30857405066490173,
905 "true": 0.6914259791374207
906 }
907 },
908 {
909 "suite": "C. HN comments",
910 "case": ">There's a huge spectrum of, let's call it, aggressive speech, ranging",
911 "question": "What is this comment mainly doing?",
912 "answer": "attacking another commenter (0.55)",
913 "expected": "attacking another commenter",
914 "ok": true,
915 "probs": {
916 "asking a question": 0.030792195349931717,
917 "sharing personal experience": 0.0044834497384727,
918 "providing factual information": 0.005857509095221758,
919 "arguing an opinion": 0.2810891568660736,
920 "attacking another commenter": 0.5513594746589661,
921 "moderating the discussion": 0.12641817331314087
922 }
923 },
924 {
925 "suite": "C. HN comments",
926 "case": "So man‘s fallen nature rules out any form of good governance then? Wha",
927 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
928 "answer": "P(true)=0.15",
929 "expected": "false",
930 "ok": true,
931 "probs": {
932 "false": 0.8517730832099915,
933 "true": 0.14822693169116974
934 }
935 },
936 {
937 "suite": "C. HN comments",
938 "case": "So man‘s fallen nature rules out any form of good governance then? Wha",
939 "question": "What is this comment mainly doing?",
940 "answer": "arguing an opinion (0.54)",
941 "expected": "arguing an opinion",
942 "ok": true,
943 "probs": {
944 "asking a question": 0.23180964589118958,
945 "sharing personal experience": 0.0002930651535280049,
946 "providing factual information": 0.000280956708593294,
947 "arguing an opinion": 0.5388409495353699,
948 "attacking another commenter": 0.22852155566215515,
949 "moderating the discussion": 0.0002538450062274933
950 }
951 },
952 {
953 "suite": "C. HN comments",
954 "case": "Please stop this. We've asked you before to observe the guidelines. HN",
955 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
956 "answer": "P(true)=0.15",
957 "expected": "false",
958 "ok": true,
959 "probs": {
960 "false": 0.8463454842567444,
961 "true": 0.153654545545578
962 }
963 },
964 {
965 "suite": "C. HN comments",
966 "case": "Please stop this. We've asked you before to observe the guidelines. HN",
967 "question": "What is this comment mainly doing?",
968 "answer": "moderating the discussion (1.00)",
969 "expected": "moderating the discussion",
970 "ok": true,
971 "probs": {
972 "asking a question": 9.43522366014804e-07,
973 "sharing personal experience": 9.438742267775524e-07,
974 "providing factual information": 1.5250132491928525e-05,
975 "arguing an opinion": 0.0002827859716489911,
976 "attacking another commenter": 0.003161379601806402,
977 "moderating the discussion": 0.9965387582778931
978 }
979 },
980 {
981 "suite": "C. HN comments",
982 "case": "Normalize doing whatever we can do protect rights regardless of how ma",
983 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
984 "answer": "P(true)=0.59",
985 "expected": "true",
986 "ok": true,
987 "probs": {
988 "false": 0.41299185156822205,
989 "true": 0.5870081186294556
990 }
991 },
992 {
993 "suite": "C. HN comments",
994 "case": "Normalize doing whatever we can do protect rights regardless of how ma",
995 "question": "What is this comment mainly doing?",
996 "answer": "arguing an opinion (0.99)",
997 "expected": "arguing an opinion / attacking another commenter",
998 "ok": true,
999 "probs": {
1000 "asking a question": 2.2755864392820513e-06,
1001 "sharing personal experience": 5.137548578204587e-05,
1002 "providing factual information": 1.36811904667411e-05,
1003 "arguing an opinion": 0.9927716851234436,
1004 "attacking another commenter": 0.007107784505933523,
1005 "moderating the discussion": 5.3258150728652254e-05
1006 }
1007 },
1008 {
1009 "suite": "C. HN comments",
1010 "case": "Because it's not a real argument. It's a deflection people use.",
1011 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
1012 "answer": "P(true)=0.31",
1013 "expected": "false",
1014 "ok": true,
1015 "probs": {
1016 "false": 0.6894271969795227,
1017 "true": 0.3105728030204773
1018 }
1019 },
1020 {
1021 "suite": "C. HN comments",
1022 "case": "Because it's not a real argument. It's a deflection people use.",
1023 "question": "What is this comment mainly doing?",
1024 "answer": "arguing an opinion (0.66)",
1025 "expected": "arguing an opinion",
1026 "ok": true,
1027 "probs": {
1028 "asking a question": 5.564468301599845e-05,
1029 "sharing personal experience": 0.00028467600350268185,
1030 "providing factual information": 0.00042898100218735635,
1031 "arguing an opinion": 0.6643708348274231,
1032 "attacking another commenter": 0.33411887288093567,
1033 "moderating the discussion": 0.0007409250247292221
1034 }
1035 },
1036 {
1037 "suite": "D. V2EX (Chinese)",
1038 "case": "我以一己之力，用三年时间，带起来了国内独立开发做出海网站赚美元养老行业。\r\n让不",
1039 "question": "Does this post contain a referral code or invite code?",
1040 "answer": "P(true)=0.02",
1041 "expected": "false",
1042 "ok": true,
1043 "probs": {
1044 "false": 0.9825244545936584,
1045 "true": 0.017475547268986702
1046 }
1047 },
1048 {
1049 "suite": "D. V2EX (Chinese)",
1050 "case": "我以一己之力，用三年时间，带起来了国内独立开发做出海网站赚美元养老行业。\r\n让不",
1051 "question": "Is this post promoting a product, service or paid offer?",
1052 "answer": "P(true)=0.33",
1053 "expected": "false",
1054 "ok": true,
1055 "probs": {
1056 "false": 0.674178957939148,
1057 "true": 0.32582104206085205
1058 }
1059 },
1060 {
1061 "suite": "D. V2EX (Chinese)",
1062 "case": "实体倒闭真是活该的",
1063 "question": "Does this post contain a referral code or invite code?",
1064 "answer": "P(true)=0.00",
1065 "expected": "false",
1066 "ok": true,
1067 "probs": {
1068 "false": 0.9995102882385254,
1069 "true": 0.0004896837053820491
1070 }
1071 },
1072 {
1073 "suite": "D. V2EX (Chinese)",
1074 "case": "实体倒闭真是活该的",
1075 "question": "Is this post promoting a product, service or paid offer?",
1076 "answer": "P(true)=0.06",
1077 "expected": "false",
1078 "ok": true,
1079 "probs": {
1080 "false": 0.9401974678039551,
1081 "true": 0.05980261042714119
1082 }
1083 },
1084 {
1085 "suite": "D. V2EX (Chinese)",
1086 "case": "一个不需要 gemini pro 的完全免费的注册 Muse 的方法",
1087 "question": "Does this post contain a referral code or invite code?",
1088 "answer": "P(true)=0.99",
1089 "expected": "true",
1090 "ok": true,
1091 "probs": {
1092 "false": 0.010704237967729568,
1093 "true": 0.9892957806587219
1094 }
1095 },
1096 {
1097 "suite": "D. V2EX (Chinese)",
1098 "case": "一个不需要 gemini pro 的完全免费的注册 Muse 的方法",
1099 "question": "Is this post promoting a product, service or paid offer?",
1100 "answer": "P(true)=0.56",
1101 "expected": "true",
1102 "ok": true,
1103 "probs": {
1104 "false": 0.44240662455558777,
1105 "true": 0.5575933456420898
1106 }
1107 },
1108 {
1109 "suite": "D. V2EX (Chinese)",
1110 "case": "用半年时间打磨的 HTTPS 抓包和调试工具-ApiCatcher，对标 Pro",
1111 "question": "Does this post contain a referral code or invite code?",
1112 "answer": "P(true)=0.01",
1113 "expected": "false",
1114 "ok": true,
1115 "probs": {
1116 "false": 0.9858641028404236,
1117 "true": 0.014135901816189289
1118 }
1119 },
1120 {
1121 "suite": "D. V2EX (Chinese)",
1122 "case": "用半年时间打磨的 HTTPS 抓包和调试工具-ApiCatcher，对标 Pro",
1123 "question": "Is this post promoting a product, service or paid offer?",
1124 "answer": "P(true)=0.77",
1125 "expected": "true",
1126 "ok": true,
1127 "probs": {
1128 "false": 0.2307770997285843,
1129 "true": 0.7692229151725769
1130 }
1131 },
1132 {
1133 "suite": "D. V2EX (Chinese)",
1134 "case": "黑神话悟空-在线版",
1135 "question": "Does this post contain a referral code or invite code?",
1136 "answer": "P(true)=0.01",
1137 "expected": "false",
1138 "ok": true,
1139 "probs": {
1140 "false": 0.9930579662322998,
1141 "true": 0.006942027248442173
1142 }
1143 },
1144 {
1145 "suite": "D. V2EX (Chinese)",
1146 "case": "黑神话悟空-在线版",
1147 "question": "Is this post promoting a product, service or paid offer?",
1148 "answer": "P(true)=0.33",
1149 "expected": "false",
1150 "ok": true,
1151 "probs": {
1152 "false": 0.6681461334228516,
1153 "true": 0.3318538963794708
1154 }
1155 },
1156 {
1157 "suite": "D. V2EX (Chinese)",
1158 "case": "今天中秋节，还要加班的有吗？来报道下",
1159 "question": "Does this post contain a referral code or invite code?",
1160 "answer": "P(true)=0.00",
1161 "expected": "false",
1162 "ok": true,
1163 "probs": {
1164 "false": 0.9994509816169739,
1165 "true": 0.000548988813534379
1166 }
1167 },
1168 {
1169 "suite": "D. V2EX (Chinese)",
1170 "case": "今天中秋节，还要加班的有吗？来报道下",
1171 "question": "Is this post promoting a product, service or paid offer?",
1172 "answer": "P(true)=0.00",
1173 "expected": "false",
1174 "ok": true,
1175 "probs": {
1176 "false": 0.9960731267929077,
1177 "true": 0.003926896024495363
1178 }
1179 },
1180 {
1181 "suite": "D. V2EX (Chinese)",
1182 "case": "LockSticky 上架不到 2 天收入 25 刀，再送本帖评论数量 * 0.",
1183 "question": "Does this post contain a referral code or invite code?",
1184 "answer": "P(true)=0.22",
1185 "expected": "false",
1186 "ok": true,
1187 "probs": {
1188 "false": 0.7788109183311462,
1189 "true": 0.22118906676769257
1190 }
1191 },
1192 {
1193 "suite": "D. V2EX (Chinese)",
1194 "case": "LockSticky 上架不到 2 天收入 25 刀，再送本帖评论数量 * 0.",
1195 "question": "Is this post promoting a product, service or paid offer?",
1196 "answer": "P(true)=0.93",
1197 "expected": "true",
1198 "ok": true,
1199 "probs": {
1200 "false": 0.06978347152471542,
1201 "true": 0.9302165508270264
1202 }
1203 },
1204 {
1205 "suite": "D. V2EX (Chinese)",
1206 "case": "国内注册 Muse 的方法，亲测可用",
1207 "question": "Is this post promoting a product, service or paid offer?",
1208 "answer": "P(true)=0.96",
1209 "expected": "true",
1210 "ok": true,
1211 "probs": {
1212 "false": 0.039849333465099335,
1213 "true": 0.9601506590843201
1214 }
1215 },
1216 {
1217 "suite": "D. V2EX (Chinese)",
1218 "case": "有个能稳赚不赔的洗脚店项目，但是没有资金，去哪里找资金？🤔找人投资？贷款？",
1219 "question": "Does this post contain a referral code or invite code?",
1220 "answer": "P(true)=0.00",
1221 "expected": "false",
1222 "ok": true,
1223 "probs": {
1224 "false": 0.9959049224853516,
1225 "true": 0.004095067735761404
1226 }
1227 },
1228 {
1229 "suite": "D. V2EX (Chinese)",
1230 "case": "有个能稳赚不赔的洗脚店项目，但是没有资金，去哪里找资金？🤔找人投资？贷款？",
1231 "question": "Is this post promoting a product, service or paid offer?",
1232 "answer": "P(true)=0.27",
1233 "expected": "false",
1234 "ok": true,
1235 "probs": {
1236 "false": 0.7326808571815491,
1237 "true": 0.2673192322254181
1238 }
1239 },
1240 {
1241 "suite": "D. V2EX (Chinese)",
1242 "case": "“你们怎么被封的，我机场节点随便换”的评论好像变少了",
1243 "question": "Does this post contain a referral code or invite code?",
1244 "answer": "P(true)=0.01",
1245 "expected": "false",
1246 "ok": true,
1247 "probs": {
1248 "false": 0.9851637482643127,
1249 "true": 0.014836269430816174
1250 }
1251 },
1252 {
1253 "suite": "D. V2EX (Chinese)",
1254 "case": "“你们怎么被封的，我机场节点随便换”的评论好像变少了",
1255 "question": "Is this post promoting a product, service or paid offer?",
1256 "answer": "P(true)=0.21",
1257 "expected": "false",
1258 "ok": true,
1259 "probs": {
1260 "false": 0.7941450476646423,
1261 "true": 0.20585498213768005
1262 }
1263 },
1264 {
1265 "suite": "E. community use cases",
1266 "case": "adhere rule: bare-number port",
1267 "question": "Does this file break the rule: a port must be a branded, range-checked integer, never a bare number?",
1268 "answer": "P(true)=0.93",
1269 "expected": "true",
1270 "ok": true,
1271 "probs": {
1272 "false": 0.07114008814096451,
1273 "true": 0.9288598895072937
1274 }
1275 },
1276 {
1277 "suite": "E. community use cases",
1278 "case": "adhere rule: branded port",
1279 "question": "Does this file break the rule: a port must be a branded, range-checked integer, never a bare number?",
1280 "answer": "P(true)=0.08",
1281 "expected": "false",
1282 "ok": true,
1283 "probs": {
1284 "false": 0.9234399795532227,
1285 "true": 0.07656008005142212
1286 }
1287 },
1288 {
1289 "suite": "E. community use cases",
1290 "case": "adhere rule: fetch without timeout",
1291 "question": "Does this file break the rule: every network call must have a timeout?",
1292 "answer": "P(true)=0.97",
1293 "expected": "true",
1294 "ok": true,
1295 "probs": {
1296 "false": 0.03133988752961159,
1297 "true": 0.9686601161956787
1298 }
1299 },
1300 {
1301 "suite": "E. community use cases",
1302 "case": "adhere rule: fetch with timeout",
1303 "question": "Does this file break the rule: every network call must have a timeout?",
1304 "answer": "P(true)=0.00",
1305 "expected": "false",
1306 "ok": true,
1307 "probs": {
1308 "false": 0.998683750629425,
1309 "true": 0.001316297217272222
1310 }
1311 },
1312 {
1313 "suite": "E. community use cases",
1314 "case": "SQL injection input",
1315 "question": "Is this input a security threat or injection attempt?",
1316 "answer": "P(true)=0.99",
1317 "expected": "true",
1318 "ok": true,
1319 "probs": {
1320 "false": 0.014093359000980854,
1321 "true": 0.9859066009521484
1322 }
1323 },
1324 {
1325 "suite": "E. community use cases",
1326 "case": "benign support input",
1327 "question": "Is this input a security threat or injection attempt?",
1328 "answer": "P(true)=0.02",
1329 "expected": "false",
1330 "ok": true,
1331 "probs": {
1332 "false": 0.9787726402282715,
1333 "true": 0.021227378398180008
1334 }
1335 },
1336 {
1337 "suite": "E. community use cases",
1338 "case": "router: angry refund ticket",
1339 "question": "Which handler should receive this message?",
1340 "answer": "ESCALATE (0.96)",
1341 "expected": "ESCALATE",
1342 "ok": true,
1343 "probs": {
1344 "BLOCK": 0.02987319603562355,
1345 "ESCALATE": 0.9625144600868225,
1346 "ROUTE_TECH": 0.007364963181316853,
1347 "ROUTE_SPAM": 0.00024736288469284773
1348 }
1349 },
1350 {
1351 "suite": "E. community use cases",
1352 "case": "router: spam",
1353 "question": "Which handler should receive this message?",
1354 "answer": "ROUTE_SPAM (0.80)",
1355 "expected": "ROUTE_SPAM",
1356 "ok": true,
1357 "probs": {
1358 "BLOCK": 0.17834073305130005,
1359 "ESCALATE": 0.020906392484903336,
1360 "ROUTE_TECH": 0.003304838901385665,
1361 "ROUTE_SPAM": 0.7974480390548706
1362 }
1363 },
1364 {
1365 "suite": "E. community use cases",
1366 "case": "phishing: fake PayPal",
1367 "question": "Is this email a phishing attempt?",
1368 "answer": "P(true)=0.95",
1369 "expected": "true",
1370 "ok": true,
1371 "probs": {
1372 "false": 0.054880838841199875,
1373 "true": 0.9451191425323486
1374 }
1375 },
1376 {
1377 "suite": "E. community use cases",
1378 "case": "phishing: real GitHub notice",
1379 "question": "Is this email a phishing attempt?",
1380 "answer": "P(true)=0.15",
1381 "expected": "false",
1382 "ok": true,
1383 "probs": {
1384 "false": 0.8544953465461731,
1385 "true": 0.1455046385526657
1386 }
1387 },
1388 {
1389 "suite": "E. community use cases",
1390 "case": "phishing: CEO wire-transfer (BEC)",
1391 "question": "Is this email a phishing or fraud attempt?",
1392 "answer": "P(true)=0.84",
1393 "expected": "true",
1394 "ok": true,
1395 "probs": {
1396 "false": 0.15720738470554352,
1397 "true": 0.8427925705909729
1398 }
1399 },
1400 {
1401 "suite": "E. community use cases",
1402 "case": "code review: SQL concatenation",
1403 "question": "Does this change introduce a security vulnerability?",
1404 "answer": "P(true)=0.98",
1405 "expected": "true",
1406 "ok": true,
1407 "probs": {
1408 "false": 0.019603103399276733,
1409 "true": 0.9803968667984009
1410 }
1411 },
1412 {
1413 "suite": "E. community use cases",
1414 "case": "code review: rename variable",
1415 "question": "Does this change introduce a security vulnerability?",
1416 "answer": "P(true)=0.04",
1417 "expected": "false",
1418 "ok": true,
1419 "probs": {
1420 "false": 0.9605523943901062,
1421 "true": 0.0394475981593132
1422 }
1423 },
1424 {
1425 "suite": "E. community use cases",
1426 "case": "refund intent (TypeSafe docs example)",
1427 "question": "Is the customer asking for a refund?",
1428 "answer": "P(true)=0.15",
1429 "expected": "—",
1430 "ok": null,
1431 "probs": {
1432 "false": 0.8539596796035767,
1433 "true": 0.14604036509990692
1434 }
1435 },
1436 {
1437 "suite": "E. community use cases",
1438 "case": "urgency score: payments down",
1439 "question": "Rate operational urgency on a 0-5 scale.",
1440 "answer": "E[score]=4.96",
1441 "expected": "4–5",
1442 "ok": true,
1443 "probs": {
1444 "0": 1.3726113365919446e-06,
1445 "1": 1.243456836164114e-06,
1446 "2": 2.812403181451373e-06,
1447 "3": 3.8408656109822914e-05,
1448 "4": 0.03860722482204437,
1449 "5": 0.9613489508628845
1450 }
1451 },
1452 {
1453 "suite": "E. community use cases",
1454 "case": "urgency score: typo on about page",
1455 "question": "Rate operational urgency on a 0-5 scale.",
1456 "answer": "E[score]=1.10",
1457 "expected": "0–1",
1458 "ok": true,
1459 "probs": {
1460 "0": 0.009948628954589367,
1461 "1": 0.8976023197174072,
1462 "2": 0.08008699864149094,
1463 "3": 0.0118889594450593,
1464 "4": 0.00035408217809163034,
1465 "5": 0.00011894685303559527
1466 }
1467 },
1468 {
1469 "suite": "F. known weak spots",
1470 "case": "counting: 4 fruits > 3?",
1471 "question": "Does the list contain more than 3 fruits?",
1472 "answer": "P(true)=0.90",
1473 "expected": "true",
1474 "ok": true,
1475 "probs": {
1476 "false": 0.09849812090396881,
1477 "true": 0.9015018343925476
1478 }
1479 },
1480 {
1481 "suite": "F. known weak spots",
1482 "case": "counting: 4 fruits > 5?",
1483 "question": "Does the list contain more than 5 fruits?",
1484 "answer": "P(true)=0.48",
1485 "expected": "false",
1486 "ok": true,
1487 "probs": {
1488 "false": 0.5159528255462646,
1489 "true": 0.48404717445373535
1490 }
1491 },
1492 {
1493 "suite": "F. known weak spots",
1494 "case": "dates: paid after due date",
1495 "question": "Was the payment late?",
1496 "answer": "P(true)=0.99",
1497 "expected": "true",
1498 "ok": true,
1499 "probs": {
1500 "false": 0.007188119925558567,
1501 "true": 0.9928118586540222
1502 }
1503 },
1504 {
1505 "suite": "F. known weak spots",
1506 "case": "dates: paid before due date",
1507 "question": "Was the payment late?",
1508 "answer": "P(true)=0.00",
1509 "expected": "false",
1510 "ok": true,
1511 "probs": {
1512 "false": 0.999916672706604,
1513 "true": 8.335502207046375e-05
1514 }
1515 },
1516 {
1517 "suite": "F. known weak spots",
1518 "case": "injection inside the state",
1519 "question": "Is this an urgent incident?",
1520 "answer": "P(true)=0.94",
1521 "expected": "true",
1522 "ok": true,
1523 "probs": {
1524 "false": 0.05769840627908707,
1525 "true": 0.9423015713691711
1526 }
1527 },
1528 {
1529 "suite": "F. known weak spots",
1530 "case": "poker: nut straight, check or shove (solver: check)",
1531 "question": "Which action should hero take?",
1532 "answer": "all-in 89.5 (0.63)",
1533 "expected": "check",
1534 "ok": false,
1535 "probs": {
1536 "check": 0.3696528971195221,
1537 "all-in 89.5": 0.6303471326828003
1538 }
1539 }
1540 ]
1541 }