reports/realworld_9b.json
48.3 KB · 1541 lines · json Raw
1 {
2 "bench": {
3 "seconds": 41.523975133895874,
4 "decisions": 14400,
5 "rq1_acc_by_k": {
6 "2": 0.8675,
7 "4": 0.77375,
8 "8": 0.735,
9 "16": 0.69375
10 },
11 "rq1_acc_by_domain_k16": {
12 "clinc": 0.875,
13 "dbpedia": 0.855,
14 "goemotions": 0.325,
15 "mtop": 0.72
16 },
17 "rq3": {
18 "2": {
19 "near": 0.9775,
20 "far": 1.0
21 },
22 "4": {
23 "near": 0.955,
24 "far": 0.9925
25 },
26 "8": {
27 "near": 0.91,
28 "far": 0.985
29 },
30 "16": {
31 "near": 0.875,
32 "far": 0.975
33 }
34 },
35 "rq2_flip_k16_item_any_change": 0.195625,
36 "rq2_flip_k16_per_decision_vs_first": 0.1146875,
37 "rq2_flip_by_domain_tier": {
38 "clinc|far": 0.01625,
39 "clinc|near": 0.065,
40 "fintopic|far": 0.02125,
41 "fintopic|near": 0.13,
42 "goemotions|far": 0.23625,
43 "goemotions|near": 0.3275,
44 "mtop|far": 0.01625,
45 "mtop|near": 0.105
46 },
47 "jev": {
48 "rq1_acc_by_k": {
49 "2": 0.89,
50 "4": 0.8013,
51 "8": 0.7825,
52 "16": 0.7688
53 },
54 "rq1_acc_by_domain_k16": {
55 "clinc": 0.945,
56 "dbpedia": 0.9,
57 "goemotions": 0.47,
58 "mtop": 0.76
59 },
60 "rq3": {
61 "2": {
62 "near": 0.985,
63 "far": 1.0
64 },
65 "4": {
66 "near": 0.97,
67 "far": 0.99
68 },
69 "8": {
70 "near": 0.9325,
71 "far": 0.98
72 },
73 "16": {
74 "near": 0.9125,
75 "far": 0.985
76 }
77 },
78 "rq2_flip_k16": 0.07,
79 "rq2_flip_by_domain_tier_k16": {
80 "clinc|far": 0.02,
81 "clinc|near": 0.015,
82 "fintopic|far": 0.02,
83 "fintopic|near": 0.08,
84 "goemotions|far": 0.105,
85 "goemotions|near": 0.235,
86 "mtop|far": 0.005,
87 "mtop|near": 0.08
88 }
89 }
90 },
91 "cases": [
92 {
93 "suite": "B. HN front page",
94 "case": "F-Droid 2.0",
95 "question": "Which topic is this story mainly about?",
96 "answer": "open-source software (0.82)",
97 "expected": "open-source software",
98 "ok": true,
99 "probs": {
100 "open-source software": 0.8154270648956299,
101 "programming languages & tools": 0.0059571778401732445,
102 "AI / machine learning": 0.0007867903914302588,
103 "security & privacy": 0.17494900524616241,
104 "hardware & retro computing": 0.001782198785804212,
105 "science & health": 0.0004612301418092102,
106 "automotive & energy": 0.00016704178415238857,
107 "politics & government": 0.00010319145076209679,
108 "business & finance": 8.154642273439094e-05,
109 "culture & lifestyle": 0.0002848256553988904
110 }
111 },
112 {
113 "suite": "B. HN front page",
114 "case": "F-Droid 2.0",
115 "question": "Is this story about artificial intelligence or machine learning?",
116 "answer": "P(true)=0.07",
117 "expected": "false",
118 "ok": true,
119 "probs": {
120 "false": 0.9257309436798096,
121 "true": 0.0742691233754158
122 }
123 },
124 {
125 "suite": "B. HN front page",
126 "case": "Dutch governments builds alternative for Microsoft based on NixOS",
127 "question": "Which topic is this story mainly about?",
128 "answer": "politics & government (0.53)",
129 "expected": "open-source software / politics & government",
130 "ok": true,
131 "probs": {
132 "open-source software": 0.39049091935157776,
133 "programming languages & tools": 0.06605563312768936,
134 "AI / machine learning": 0.00036396001814864576,
135 "security & privacy": 0.008612526580691338,
136 "hardware & retro computing": 0.0018591288244351745,
137 "science & health": 0.0005603310419246554,
138 "automotive & energy": 0.00018504075706005096,
139 "politics & government": 0.5314207077026367,
140 "business & finance": 0.00023308150412049145,
141 "culture & lifestyle": 0.00021863346046302468
142 }
143 },
144 {
145 "suite": "B. HN front page",
146 "case": "Dutch governments builds alternative for Microsoft based on NixOS",
147 "question": "Is this story about artificial intelligence or machine learning?",
148 "answer": "P(true)=0.02",
149 "expected": "false",
150 "ok": true,
151 "probs": {
152 "false": 0.975690484046936,
153 "true": 0.02430950291454792
154 }
155 },
156 {
157 "suite": "B. HN front page",
158 "case": "Two-tier encryption in the UK",
159 "question": "Which topic is this story mainly about?",
160 "answer": "security & privacy (0.87)",
161 "expected": "politics & government / security & privacy",
162 "ok": true,
163 "probs": {
164 "open-source software": 0.00020718226733151823,
165 "programming languages & tools": 0.00033174952841363847,
166 "AI / machine learning": 0.0001234175724675879,
167 "security & privacy": 0.8726596832275391,
168 "hardware & retro computing": 0.0005030221655033529,
169 "science & health": 8.82171152625233e-05,
170 "automotive & energy": 2.449350358801894e-05,
171 "politics & government": 0.12594789266586304,
172 "business & finance": 8.537234680261463e-05,
173 "culture & lifestyle": 2.8984964956180193e-05
174 }
175 },
176 {
177 "suite": "B. HN front page",
178 "case": "Two-tier encryption in the UK",
179 "question": "Is this story about artificial intelligence or machine learning?",
180 "answer": "P(true)=0.07",
181 "expected": "false",
182 "ok": true,
183 "probs": {
184 "false": 0.9297435283660889,
185 "true": 0.07025653123855591
186 }
187 },
188 {
189 "suite": "B. HN front page",
190 "case": "Why is the liver so weirdly regenerative?",
191 "question": "Which topic is this story mainly about?",
192 "answer": "science & health (1.00)",
193 "expected": "science & health",
194 "ok": true,
195 "probs": {
196 "open-source software": 1.1553935109986924e-05,
197 "programming languages & tools": 1.9779134163400158e-05,
198 "AI / machine learning": 6.116784061305225e-05,
199 "security & privacy": 2.6431158403283916e-05,
200 "hardware & retro computing": 6.308336651272839e-06,
201 "science & health": 0.9998742341995239,
202 "automotive & energy": 5.673905789649325e-08,
203 "politics & government": 8.447337052075454e-09,
204 "business & finance": 6.464690560648023e-09,
205 "culture & lifestyle": 3.109982458227023e-07
206 }
207 },
208 {
209 "suite": "B. HN front page",
210 "case": "Why is the liver so weirdly regenerative?",
211 "question": "Is this story about artificial intelligence or machine learning?",
212 "answer": "P(true)=0.07",
213 "expected": "false",
214 "ok": true,
215 "probs": {
216 "false": 0.932094156742096,
217 "true": 0.06790585070848465
218 }
219 },
220 {
221 "suite": "B. HN front page",
222 "case": "Toyota is taking the Corolla electric",
223 "question": "Which topic is this story mainly about?",
224 "answer": "automotive & energy (1.00)",
225 "expected": "automotive & energy",
226 "ok": true,
227 "probs": {
228 "open-source software": 9.605043942428892e-07,
229 "programming languages & tools": 3.825568455795292e-06,
230 "AI / machine learning": 1.0744451174105052e-05,
231 "security & privacy": 7.32915141270496e-05,
232 "hardware & retro computing": 1.3163899893697817e-05,
233 "science & health": 3.244264803470287e-07,
234 "automotive & energy": 0.9998805522918701,
235 "politics & government": 1.959028850251343e-08,
236 "business & finance": 1.658730070630554e-05,
237 "culture & lifestyle": 4.3374612346269714e-07
238 }
239 },
240 {
241 "suite": "B. HN front page",
242 "case": "Toyota is taking the Corolla electric",
243 "question": "Is this story about artificial intelligence or machine learning?",
244 "answer": "P(true)=0.05",
245 "expected": "false",
246 "ok": true,
247 "probs": {
248 "false": 0.9540500640869141,
249 "true": 0.04594998061656952
250 }
251 },
252 {
253 "suite": "B. HN front page",
254 "case": "Rails World 2026 Opening Keynote [video]",
255 "question": "Which topic is this story mainly about?",
256 "answer": "programming languages & tools (0.85)",
257 "expected": "programming languages & tools",
258 "ok": true,
259 "probs": {
260 "open-source software": 0.11634442955255508,
261 "programming languages & tools": 0.8470632433891296,
262 "AI / machine learning": 0.01612945646047592,
263 "security & privacy": 0.004623000510036945,
264 "hardware & retro computing": 0.002891573589295149,
265 "science & health": 0.0008532701176591218,
266 "automotive & energy": 0.0002424292906653136,
267 "politics & government": 0.0001360717142233625,
268 "business & finance": 0.0005236657452769578,
269 "culture & lifestyle": 0.011192874982953072
270 }
271 },
272 {
273 "suite": "B. HN front page",
274 "case": "Rails World 2026 Opening Keynote [video]",
275 "question": "Is this story about artificial intelligence or machine learning?",
276 "answer": "P(true)=0.11",
277 "expected": "false",
278 "ok": true,
279 "probs": {
280 "false": 0.8922533392906189,
281 "true": 0.1077466607093811
282 }
283 },
284 {
285 "suite": "B. HN front page",
286 "case": "Opus 5.5 is good at explainer videos",
287 "question": "Which topic is this story mainly about?",
288 "answer": "AI / machine learning (0.78)",
289 "expected": "AI / machine learning",
290 "ok": true,
291 "probs": {
292 "open-source software": 0.03562888503074646,
293 "programming languages & tools": 0.13412527740001678,
294 "AI / machine learning": 0.7816400527954102,
295 "security & privacy": 0.010795998387038708,
296 "hardware & retro computing": 0.01016384270042181,
297 "science & health": 0.0032580315601080656,
298 "automotive & energy": 0.001083938404917717,
299 "politics & government": 0.0004676562384702265,
300 "business & finance": 0.010733872652053833,
301 "culture & lifestyle": 0.01210243534296751
302 }
303 },
304 {
305 "suite": "B. HN front page",
306 "case": "Opus 5.5 is good at explainer videos",
307 "question": "Is this story about artificial intelligence or machine learning?",
308 "answer": "P(true)=0.62",
309 "expected": "true",
310 "ok": true,
311 "probs": {
312 "false": 0.3843480050563812,
313 "true": 0.6156519651412964
314 }
315 },
316 {
317 "suite": "B. HN front page",
318 "case": "Fearless SIMD v1.0",
319 "question": "Which topic is this story mainly about?",
320 "answer": "programming languages & tools (0.72)",
321 "expected": "programming languages & tools",
322 "ok": true,
323 "probs": {
324 "open-source software": 0.14773355424404144,
325 "programming languages & tools": 0.7175008058547974,
326 "AI / machine learning": 0.015606332570314407,
327 "security & privacy": 0.028633734211325645,
328 "hardware & retro computing": 0.08832192420959473,
329 "science & health": 0.001056667766533792,
330 "automotive & energy": 0.0005287938984110951,
331 "politics & government": 0.00017177339759655297,
332 "business & finance": 8.216068818001077e-05,
333 "culture & lifestyle": 0.00036429284955374897
334 }
335 },
336 {
337 "suite": "B. HN front page",
338 "case": "Fearless SIMD v1.0",
339 "question": "Is this story about artificial intelligence or machine learning?",
340 "answer": "P(true)=0.10",
341 "expected": "false",
342 "ok": true,
343 "probs": {
344 "false": 0.9034923911094666,
345 "true": 0.09650754928588867
346 }
347 },
348 {
349 "suite": "B. HN front page",
350 "case": "Pentium II at 600Mhz with Voodoo 3 Emulated on 86Box with M6 Mac Mini",
351 "question": "Which topic is this story mainly about?",
352 "answer": "hardware & retro computing (1.00)",
353 "expected": "hardware & retro computing",
354 "ok": true,
355 "probs": {
356 "open-source software": 0.0004247011383995414,
357 "programming languages & tools": 0.00030374692869372666,
358 "AI / machine learning": 5.5250733566936105e-05,
359 "security & privacy": 0.0008668869850225747,
360 "hardware & retro computing": 0.9983355402946472,
361 "science & health": 1.2486439118220005e-06,
362 "automotive & energy": 3.382928355222248e-07,
363 "politics & government": 2.778048724394466e-07,
364 "business & finance": 3.2886890721783857e-07,
365 "culture & lifestyle": 1.1785299648181535e-05
366 }
367 },
368 {
369 "suite": "B. HN front page",
370 "case": "Pentium II at 600Mhz with Voodoo 3 Emulated on 86Box with M6 Mac Mini",
371 "question": "Is this story about artificial intelligence or machine learning?",
372 "answer": "P(true)=0.02",
373 "expected": "false",
374 "ok": true,
375 "probs": {
376 "false": 0.97784024477005,
377 "true": 0.022159690037369728
378 }
379 },
380 {
381 "suite": "B. HN front page",
382 "case": "The Mafia may be keeping fentanyl out of Italy",
383 "question": "Which topic is this story mainly about?",
384 "answer": "science & health (0.51)",
385 "expected": "politics & government / science & health",
386 "ok": true,
387 "probs": {
388 "open-source software": 0.000999127165414393,
389 "programming languages & tools": 0.0009710687445476651,
390 "AI / machine learning": 0.0013503868831321597,
391 "security & privacy": 0.07248688489198685,
392 "hardware & retro computing": 0.0012125493958592415,
393 "science & health": 0.5148013830184937,
394 "automotive & energy": 0.0003496279241517186,
395 "politics & government": 0.3886215388774872,
396 "business & finance": 0.01718498393893242,
397 "culture & lifestyle": 0.002022478263825178
398 }
399 },
400 {
401 "suite": "B. HN front page",
402 "case": "The Mafia may be keeping fentanyl out of Italy",
403 "question": "Is this story about artificial intelligence or machine learning?",
404 "answer": "P(true)=0.04",
405 "expected": "false",
406 "ok": true,
407 "probs": {
408 "false": 0.9632319808006287,
409 "true": 0.036768049001693726
410 }
411 },
412 {
413 "suite": "B. HN front page",
414 "case": "Oracle on the hook to pay data centre investors even if site has no electricity",
415 "question": "Which topic is this story mainly about?",
416 "answer": "business & finance (0.98)",
417 "expected": "business & finance",
418 "ok": true,
419 "probs": {
420 "open-source software": 3.5713914257939905e-05,
421 "programming languages & tools": 4.658008037949912e-05,
422 "AI / machine learning": 0.0005968229379504919,
423 "security & privacy": 0.0011728124227374792,
424 "hardware & retro computing": 0.0006491027306765318,
425 "science & health": 1.8485950931790285e-05,
426 "automotive & energy": 0.01592373475432396,
427 "politics & government": 3.938163354177959e-05,
428 "business & finance": 0.981508195400238,
429 "culture & lifestyle": 9.193187906930689e-06
430 }
431 },
432 {
433 "suite": "B. HN front page",
434 "case": "Oracle on the hook to pay data centre investors even if site has no electricity",
435 "question": "Is this story about artificial intelligence or machine learning?",
436 "answer": "P(true)=0.08",
437 "expected": "false",
438 "ok": true,
439 "probs": {
440 "false": 0.9209179878234863,
441 "true": 0.07908203452825546
442 }
443 },
444 {
445 "suite": "B. HN front page",
446 "case": "CVE-2025-13032: Entering and Breaking the Avast Antivirus Sandbox Part 2",
447 "question": "Which topic is this story mainly about?",
448 "answer": "security & privacy (1.00)",
449 "expected": "security & privacy",
450 "ok": true,
451 "probs": {
452 "open-source software": 9.820506966207176e-05,
453 "programming languages & tools": 0.0005866047577001154,
454 "AI / machine learning": 3.604530866141431e-05,
455 "security & privacy": 0.9992339611053467,
456 "hardware & retro computing": 4.0768292819848284e-05,
457 "science & health": 3.2882173854886787e-06,
458 "automotive & energy": 4.280073540030571e-07,
459 "politics & government": 3.482274451016565e-07,
460 "business & finance": 1.025927289788342e-07,
461 "culture & lifestyle": 2.1999134958150535e-07
462 }
463 },
464 {
465 "suite": "B. HN front page",
466 "case": "CVE-2025-13032: Entering and Breaking the Avast Antivirus Sandbox Part 2",
467 "question": "Is this story about artificial intelligence or machine learning?",
468 "answer": "P(true)=0.06",
469 "expected": "false",
470 "ok": true,
471 "probs": {
472 "false": 0.9439244866371155,
473 "true": 0.05607553943991661
474 }
475 },
476 {
477 "suite": "B. HN front page",
478 "case": "Platform-Independent SIMD in Go",
479 "question": "Which topic is this story mainly about?",
480 "answer": "programming languages & tools (0.93)",
481 "expected": "programming languages & tools",
482 "ok": true,
483 "probs": {
484 "open-source software": 0.007786287926137447,
485 "programming languages & tools": 0.9281560778617859,
486 "AI / machine learning": 0.005088189151138067,
487 "security & privacy": 0.0018334888154640794,
488 "hardware & retro computing": 0.05699412152171135,
489 "science & health": 7.78732355684042e-05,
490 "automotive & energy": 3.1681836844654754e-05,
491 "politics & government": 9.384140867041424e-06,
492 "business & finance": 5.54940424990491e-06,
493 "culture & lifestyle": 1.7241420209757052e-05
494 }
495 },
496 {
497 "suite": "B. HN front page",
498 "case": "Platform-Independent SIMD in Go",
499 "question": "Is this story about artificial intelligence or machine learning?",
500 "answer": "P(true)=0.05",
501 "expected": "false",
502 "ok": true,
503 "probs": {
504 "false": 0.9543805718421936,
505 "true": 0.04561939463019371
506 }
507 },
508 {
509 "suite": "B. HN front page",
510 "case": "Git-bug: Distributed, offline-first bug tracker embedded in Git",
511 "question": "Which topic is this story mainly about?",
512 "answer": "programming languages & tools (0.74)",
513 "expected": "open-source software / programming languages & tools",
514 "ok": true,
515 "probs": {
516 "open-source software": 0.22218094766139984,
517 "programming languages & tools": 0.7359927892684937,
518 "AI / machine learning": 0.00036510342033579946,
519 "security & privacy": 0.03976652771234512,
520 "hardware & retro computing": 0.0011527878232300282,
521 "science & health": 0.0003341646515764296,
522 "automotive & energy": 7.182945410022512e-05,
523 "politics & government": 4.062236621393822e-05,
524 "business & finance": 1.5823095964151435e-05,
525 "culture & lifestyle": 7.937061309348792e-05
526 }
527 },
528 {
529 "suite": "B. HN front page",
530 "case": "Git-bug: Distributed, offline-first bug tracker embedded in Git",
531 "question": "Is this story about artificial intelligence or machine learning?",
532 "answer": "P(true)=0.04",
533 "expected": "false",
534 "ok": true,
535 "probs": {
536 "false": 0.9633795022964478,
537 "true": 0.03662045672535896
538 }
539 },
540 {
541 "suite": "B. HN front page",
542 "case": "Using LLMs to trace alchemical knowledge and decode 17th century letters",
543 "question": "Which topic is this story mainly about?",
544 "answer": "AI / machine learning (0.98)",
545 "expected": "AI / machine learning / culture & lifestyle",
546 "ok": true,
547 "probs": {
548 "open-source software": 0.00041197906830348074,
549 "programming languages & tools": 0.0007670731865800917,
550 "AI / machine learning": 0.9801949262619019,
551 "security & privacy": 0.0023326128721237183,
552 "hardware & retro computing": 0.0007045547245070338,
553 "science & health": 0.00802137516438961,
554 "automotive & energy": 6.988887889747275e-06,
555 "politics & government": 7.936602742120158e-06,
556 "business & finance": 6.4789451243996155e-06,
557 "culture & lifestyle": 0.007546106819063425
558 }
559 },
560 {
561 "suite": "B. HN front page",
562 "case": "Using LLMs to trace alchemical knowledge and decode 17th century letters",
563 "question": "Is this story about artificial intelligence or machine learning?",
564 "answer": "P(true)=0.88",
565 "expected": "true",
566 "ok": true,
567 "probs": {
568 "false": 0.12196695804595947,
569 "true": 0.8780329823493958
570 }
571 },
572 {
573 "suite": "B. HN front page",
574 "case": "Show HN: Agentic CUDA Kernel Optimizer",
575 "question": "Which topic is this story mainly about?",
576 "answer": "programming languages & tools (0.45)",
577 "expected": "AI / machine learning / programming languages & tools",
578 "ok": true,
579 "probs": {
580 "open-source software": 0.12628887593746185,
581 "programming languages & tools": 0.4462573826313019,
582 "AI / machine learning": 0.3554892838001251,
583 "security & privacy": 0.0011440098751336336,
584 "hardware & retro computing": 0.0690179318189621,
585 "science & health": 0.0008435178315266967,
586 "automotive & energy": 0.0005016618524678051,
587 "politics & government": 0.00011135449312860146,
588 "business & finance": 9.733746264828369e-05,
589 "culture & lifestyle": 0.00024866993771865964
590 }
591 },
592 {
593 "suite": "B. HN front page",
594 "case": "Show HN: Agentic CUDA Kernel Optimizer",
595 "question": "Is this story about artificial intelligence or machine learning?",
596 "answer": "P(true)=0.61",
597 "expected": "true",
598 "ok": true,
599 "probs": {
600 "false": 0.39386624097824097,
601 "true": 0.606133759021759
602 }
603 },
604 {
605 "suite": "B. HN front page",
606 "case": "Amiga Screens: A Primer",
607 "question": "Which topic is this story mainly about?",
608 "answer": "hardware & retro computing (1.00)",
609 "expected": "hardware & retro computing",
610 "ok": true,
611 "probs": {
612 "open-source software": 0.0004919389029964805,
613 "programming languages & tools": 0.0017389487475156784,
614 "AI / machine learning": 0.0002616380515974015,
615 "security & privacy": 0.00030200896435417235,
616 "hardware & retro computing": 0.9964062571525574,
617 "science & health": 5.553035862249089e-06,
618 "automotive & energy": 2.5512617867207155e-06,
619 "politics & government": 1.445149109713384e-06,
620 "business & finance": 2.042302412519348e-06,
621 "culture & lifestyle": 0.0007875981973484159
622 }
623 },
624 {
625 "suite": "B. HN front page",
626 "case": "Amiga Screens: A Primer",
627 "question": "Is this story about artificial intelligence or machine learning?",
628 "answer": "P(true)=0.06",
629 "expected": "false",
630 "ok": true,
631 "probs": {
632 "false": 0.9388139843940735,
633 "true": 0.0611860454082489
634 }
635 },
636 {
637 "suite": "B. HN front page",
638 "case": "My weird new hobby: Wandering around Tokyo on Google Maps",
639 "question": "Which topic is this story mainly about?",
640 "answer": "culture & lifestyle (0.83)",
641 "expected": "culture & lifestyle",
642 "ok": true,
643 "probs": {
644 "open-source software": 0.0038387435488402843,
645 "programming languages & tools": 0.032091885805130005,
646 "AI / machine learning": 0.044195059686899185,
647 "security & privacy": 0.07987529784440994,
648 "hardware & retro computing": 0.003926487639546394,
649 "science & health": 0.0009113933774642646,
650 "automotive & energy": 0.00038123279227875173,
651 "politics & government": 0.0001094527033274062,
652 "business & finance": 0.0001232384383911267,
653 "culture & lifestyle": 0.834547221660614
654 }
655 },
656 {
657 "suite": "B. HN front page",
658 "case": "My weird new hobby: Wandering around Tokyo on Google Maps",
659 "question": "Is this story about artificial intelligence or machine learning?",
660 "answer": "P(true)=0.14",
661 "expected": "false",
662 "ok": true,
663 "probs": {
664 "false": 0.8575693368911743,
665 "true": 0.14243057370185852
666 }
667 },
668 {
669 "suite": "B. HN front page",
670 "case": "Topcoat is pushing the boundary of server applications with Rust",
671 "question": "Which topic is this story mainly about?",
672 "answer": "programming languages & tools (0.98)",
673 "expected": "open-source software / programming languages & tools",
674 "ok": true,
675 "probs": {
676 "open-source software": 0.019929807633161545,
677 "programming languages & tools": 0.9757442474365234,
678 "AI / machine learning": 0.0002712879213504493,
679 "security & privacy": 0.0036269184201955795,
680 "hardware & retro computing": 0.0003389274061191827,
681 "science & health": 5.418203727458604e-05,
682 "automotive & energy": 1.6128824427141808e-05,
683 "politics & government": 3.0715334560227348e-06,
684 "business & finance": 9.000385034596547e-06,
685 "culture & lifestyle": 6.4476635088794865e-06
686 }
687 },
688 {
689 "suite": "B. HN front page",
690 "case": "Topcoat is pushing the boundary of server applications with Rust",
691 "question": "Is this story about artificial intelligence or machine learning?",
692 "answer": "P(true)=0.04",
693 "expected": "false",
694 "ok": true,
695 "probs": {
696 "false": 0.9634268879890442,
697 "true": 0.0365731418132782
698 }
699 },
700 {
701 "suite": "C. HN comments",
702 "case": ">...only Signal completely opts out of iCloud... There is an iCloud op",
703 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
704 "answer": "P(true)=0.01",
705 "expected": "false",
706 "ok": true,
707 "probs": {
708 "false": 0.9876093864440918,
709 "true": 0.01239057071506977
710 }
711 },
712 {
713 "suite": "C. HN comments",
714 "case": ">...only Signal completely opts out of iCloud... There is an iCloud op",
715 "question": "What is this comment mainly doing?",
716 "answer": "providing factual information (0.69)",
717 "expected": "providing factual information",
718 "ok": true,
719 "probs": {
720 "asking a question": 0.24435074627399445,
721 "sharing personal experience": 0.016452381387352943,
722 "providing factual information": 0.6882879137992859,
723 "arguing an opinion": 0.04238937050104141,
724 "attacking another commenter": 0.006122976541519165,
725 "moderating the discussion": 0.002396611263975501
726 }
727 },
728 {
729 "suite": "C. HN comments",
730 "case": "You're not British. Probably, you've never been to the UK. Why are you",
731 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
732 "answer": "P(true)=0.73",
733 "expected": "true",
734 "ok": true,
735 "probs": {
736 "false": 0.2715883255004883,
737 "true": 0.7284116744995117
738 }
739 },
740 {
741 "suite": "C. HN comments",
742 "case": "You're not British. Probably, you've never been to the UK. Why are you",
743 "question": "What is this comment mainly doing?",
744 "answer": "attacking another commenter (0.68)",
745 "expected": "arguing an opinion / attacking another commenter",
746 "ok": true,
747 "probs": {
748 "asking a question": 0.009796771220862865,
749 "sharing personal experience": 0.09384701400995255,
750 "providing factual information": 0.0034877106081694365,
751 "arguing an opinion": 0.21461479365825653,
752 "attacking another commenter": 0.6771043539047241,
753 "moderating the discussion": 0.0011493272613734007
754 }
755 },
756 {
757 "suite": "C. HN comments",
758 "case": "> WhatsApp is similar by the way. Unless you enable E2E backups, they ",
759 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
760 "answer": "P(true)=0.02",
761 "expected": "false",
762 "ok": true,
763 "probs": {
764 "false": 0.9848233461380005,
765 "true": 0.015176604501903057
766 }
767 },
768 {
769 "suite": "C. HN comments",
770 "case": "> WhatsApp is similar by the way. Unless you enable E2E backups, they ",
771 "question": "What is this comment mainly doing?",
772 "answer": "arguing an opinion (0.57)",
773 "expected": "providing factual information",
774 "ok": false,
775 "probs": {
776 "asking a question": 0.001550900167785585,
777 "sharing personal experience": 0.007682987954467535,
778 "providing factual information": 0.4169791340827942,
779 "arguing an opinion": 0.566910982131958,
780 "attacking another commenter": 0.005863634403795004,
781 "moderating the discussion": 0.0010123294778168201
782 }
783 },
784 {
785 "suite": "C. HN comments",
786 "case": "The article mentions that Cellebrite managed to gain access to the pho",
787 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
788 "answer": "P(true)=0.05",
789 "expected": "false",
790 "ok": true,
791 "probs": {
792 "false": 0.951637864112854,
793 "true": 0.04836207255721092
794 }
795 },
796 {
797 "suite": "C. HN comments",
798 "case": "The article mentions that Cellebrite managed to gain access to the pho",
799 "question": "What is this comment mainly doing?",
800 "answer": "asking a question (0.97)",
801 "expected": "asking a question",
802 "ok": true,
803 "probs": {
804 "asking a question": 0.9671891927719116,
805 "sharing personal experience": 0.00012975525169167668,
806 "providing factual information": 0.0036078118719160557,
807 "arguing an opinion": 0.028267914429306984,
808 "attacking another commenter": 0.0006824244046583772,
809 "moderating the discussion": 0.00012290276936255395
810 }
811 },
812 {
813 "suite": "C. HN comments",
814 "case": "For anyone reading, this is entirely false. As sdcfgy says, these even",
815 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
816 "answer": "P(true)=0.11",
817 "expected": "false",
818 "ok": true,
819 "probs": {
820 "false": 0.8876181244850159,
821 "true": 0.1123819351196289
822 }
823 },
824 {
825 "suite": "C. HN comments",
826 "case": "For anyone reading, this is entirely false. As sdcfgy says, these even",
827 "question": "What is this comment mainly doing?",
828 "answer": "providing factual information (0.41)",
829 "expected": "arguing an opinion / providing factual information",
830 "ok": true,
831 "probs": {
832 "asking a question": 0.001865175785496831,
833 "sharing personal experience": 0.006318636238574982,
834 "providing factual information": 0.40936049818992615,
835 "arguing an opinion": 0.3745451271533966,
836 "attacking another commenter": 0.20073649287223816,
837 "moderating the discussion": 0.007174073718488216
838 }
839 },
840 {
841 "suite": "C. HN comments",
842 "case": "I moved my Apple account region to Spain and used a Banco Santander ba",
843 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
844 "answer": "P(true)=0.05",
845 "expected": "false",
846 "ok": true,
847 "probs": {
848 "false": 0.9504996538162231,
849 "true": 0.049500320106744766
850 }
851 },
852 {
853 "suite": "C. HN comments",
854 "case": "I moved my Apple account region to Spain and used a Banco Santander ba",
855 "question": "What is this comment mainly doing?",
856 "answer": "sharing personal experience (0.86)",
857 "expected": "sharing personal experience",
858 "ok": true,
859 "probs": {
860 "asking a question": 0.004469515755772591,
861 "sharing personal experience": 0.858604907989502,
862 "providing factual information": 0.08212896436452866,
863 "arguing an opinion": 0.04176383465528488,
864 "attacking another commenter": 0.011869320645928383,
865 "moderating the discussion": 0.0011634735856205225
866 }
867 },
868 {
869 "suite": "C. HN comments",
870 "case": "How will you insert your physical ID card into your computer to prove ",
871 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
872 "answer": "P(true)=0.34",
873 "expected": "false",
874 "ok": true,
875 "probs": {
876 "false": 0.6553356647491455,
877 "true": 0.34466442465782166
878 }
879 },
880 {
881 "suite": "C. HN comments",
882 "case": "How will you insert your physical ID card into your computer to prove ",
883 "question": "What is this comment mainly doing?",
884 "answer": "asking a question (0.47)",
885 "expected": "arguing an opinion / asking a question",
886 "ok": true,
887 "probs": {
888 "asking a question": 0.46647563576698303,
889 "sharing personal experience": 0.008892478421330452,
890 "providing factual information": 0.020052960142493248,
891 "arguing an opinion": 0.29312509298324585,
892 "attacking another commenter": 0.2061537206172943,
893 "moderating the discussion": 0.005300154443830252
894 }
895 },
896 {
897 "suite": "C. HN comments",
898 "case": ">There's a huge spectrum of, let's call it, aggressive speech, ranging",
899 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
900 "answer": "P(true)=0.67",
901 "expected": "true",
902 "ok": true,
903 "probs": {
904 "false": 0.33330148458480835,
905 "true": 0.6666985154151917
906 }
907 },
908 {
909 "suite": "C. HN comments",
910 "case": ">There's a huge spectrum of, let's call it, aggressive speech, ranging",
911 "question": "What is this comment mainly doing?",
912 "answer": "attacking another commenter (0.61)",
913 "expected": "attacking another commenter",
914 "ok": true,
915 "probs": {
916 "asking a question": 0.05586916208267212,
917 "sharing personal experience": 0.008536363951861858,
918 "providing factual information": 0.0075917914509773254,
919 "arguing an opinion": 0.24183502793312073,
920 "attacking another commenter": 0.6128338575363159,
921 "moderating the discussion": 0.07333377003669739
922 }
923 },
924 {
925 "suite": "C. HN comments",
926 "case": "So man‘s fallen nature rules out any form of good governance then? Wha",
927 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
928 "answer": "P(true)=0.30",
929 "expected": "false",
930 "ok": true,
931 "probs": {
932 "false": 0.6996371150016785,
933 "true": 0.30036285519599915
934 }
935 },
936 {
937 "suite": "C. HN comments",
938 "case": "So man‘s fallen nature rules out any form of good governance then? Wha",
939 "question": "What is this comment mainly doing?",
940 "answer": "arguing an opinion (0.62)",
941 "expected": "arguing an opinion",
942 "ok": true,
943 "probs": {
944 "asking a question": 0.029278302565217018,
945 "sharing personal experience": 0.0008145824540406466,
946 "providing factual information": 0.00114194315392524,
947 "arguing an opinion": 0.6218293905258179,
948 "attacking another commenter": 0.34623289108276367,
949 "moderating the discussion": 0.0007028732215985656
950 }
951 },
952 {
953 "suite": "C. HN comments",
954 "case": "Please stop this. We've asked you before to observe the guidelines. HN",
955 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
956 "answer": "P(true)=0.34",
957 "expected": "false",
958 "ok": true,
959 "probs": {
960 "false": 0.6596718430519104,
961 "true": 0.340328186750412
962 }
963 },
964 {
965 "suite": "C. HN comments",
966 "case": "Please stop this. We've asked you before to observe the guidelines. HN",
967 "question": "What is this comment mainly doing?",
968 "answer": "moderating the discussion (0.78)",
969 "expected": "moderating the discussion",
970 "ok": true,
971 "probs": {
972 "asking a question": 0.00019358788267709315,
973 "sharing personal experience": 0.0005689807585440576,
974 "providing factual information": 0.0017743594944477081,
975 "arguing an opinion": 0.019039953127503395,
976 "attacking another commenter": 0.198715478181839,
977 "moderating the discussion": 0.779707670211792
978 }
979 },
980 {
981 "suite": "C. HN comments",
982 "case": "Normalize doing whatever we can do protect rights regardless of how ma",
983 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
984 "answer": "P(true)=0.65",
985 "expected": "true",
986 "ok": true,
987 "probs": {
988 "false": 0.35098519921302795,
989 "true": 0.6490148305892944
990 }
991 },
992 {
993 "suite": "C. HN comments",
994 "case": "Normalize doing whatever we can do protect rights regardless of how ma",
995 "question": "What is this comment mainly doing?",
996 "answer": "arguing an opinion (0.98)",
997 "expected": "arguing an opinion / attacking another commenter",
998 "ok": true,
999 "probs": {
1000 "asking a question": 6.0618938732659444e-05,
1001 "sharing personal experience": 0.0016184309497475624,
1002 "providing factual information": 0.00039895076770335436,
1003 "arguing an opinion": 0.9779714941978455,
1004 "attacking another commenter": 0.019744962453842163,
1005 "moderating the discussion": 0.0002055476070381701
1006 }
1007 },
1008 {
1009 "suite": "C. HN comments",
1010 "case": "Because it's not a real argument. It's a deflection people use.",
1011 "question": "Does this comment insult or personally attack another person or group, rather than just arguing the point?",
1012 "answer": "P(true)=0.33",
1013 "expected": "false",
1014 "ok": true,
1015 "probs": {
1016 "false": 0.6723310351371765,
1017 "true": 0.3276689350605011
1018 }
1019 },
1020 {
1021 "suite": "C. HN comments",
1022 "case": "Because it's not a real argument. It's a deflection people use.",
1023 "question": "What is this comment mainly doing?",
1024 "answer": "attacking another commenter (0.66)",
1025 "expected": "arguing an opinion",
1026 "ok": false,
1027 "probs": {
1028 "asking a question": 0.0013444507494568825,
1029 "sharing personal experience": 0.002169741550460458,
1030 "providing factual information": 0.013073379173874855,
1031 "arguing an opinion": 0.3166387677192688,
1032 "attacking another commenter": 0.6629198789596558,
1033 "moderating the discussion": 0.0038537275977432728
1034 }
1035 },
1036 {
1037 "suite": "D. V2EX (Chinese)",
1038 "case": "我以一己之力，用三年时间，带起来了国内独立开发做出海网站赚美元养老行业。\r\n让不",
1039 "question": "Does this post contain a referral code or invite code?",
1040 "answer": "P(true)=0.04",
1041 "expected": "false",
1042 "ok": true,
1043 "probs": {
1044 "false": 0.9577800035476685,
1045 "true": 0.04222002252936363
1046 }
1047 },
1048 {
1049 "suite": "D. V2EX (Chinese)",
1050 "case": "我以一己之力，用三年时间，带起来了国内独立开发做出海网站赚美元养老行业。\r\n让不",
1051 "question": "Is this post promoting a product, service or paid offer?",
1052 "answer": "P(true)=0.22",
1053 "expected": "false",
1054 "ok": true,
1055 "probs": {
1056 "false": 0.7843812108039856,
1057 "true": 0.2156187742948532
1058 }
1059 },
1060 {
1061 "suite": "D. V2EX (Chinese)",
1062 "case": "实体倒闭真是活该的",
1063 "question": "Does this post contain a referral code or invite code?",
1064 "answer": "P(true)=0.00",
1065 "expected": "false",
1066 "ok": true,
1067 "probs": {
1068 "false": 0.9985966086387634,
1069 "true": 0.0014033587649464607
1070 }
1071 },
1072 {
1073 "suite": "D. V2EX (Chinese)",
1074 "case": "实体倒闭真是活该的",
1075 "question": "Is this post promoting a product, service or paid offer?",
1076 "answer": "P(true)=0.09",
1077 "expected": "false",
1078 "ok": true,
1079 "probs": {
1080 "false": 0.9091281294822693,
1081 "true": 0.09087185561656952
1082 }
1083 },
1084 {
1085 "suite": "D. V2EX (Chinese)",
1086 "case": "一个不需要 gemini pro 的完全免费的注册 Muse 的方法",
1087 "question": "Does this post contain a referral code or invite code?",
1088 "answer": "P(true)=1.00",
1089 "expected": "true",
1090 "ok": true,
1091 "probs": {
1092 "false": 0.004516690503805876,
1093 "true": 0.9954832792282104
1094 }
1095 },
1096 {
1097 "suite": "D. V2EX (Chinese)",
1098 "case": "一个不需要 gemini pro 的完全免费的注册 Muse 的方法",
1099 "question": "Is this post promoting a product, service or paid offer?",
1100 "answer": "P(true)=0.66",
1101 "expected": "true",
1102 "ok": true,
1103 "probs": {
1104 "false": 0.3362550437450409,
1105 "true": 0.6637449860572815
1106 }
1107 },
1108 {
1109 "suite": "D. V2EX (Chinese)",
1110 "case": "用半年时间打磨的 HTTPS 抓包和调试工具-ApiCatcher，对标 Pro",
1111 "question": "Does this post contain a referral code or invite code?",
1112 "answer": "P(true)=0.00",
1113 "expected": "false",
1114 "ok": true,
1115 "probs": {
1116 "false": 0.9963895678520203,
1117 "true": 0.0036103881429880857
1118 }
1119 },
1120 {
1121 "suite": "D. V2EX (Chinese)",
1122 "case": "用半年时间打磨的 HTTPS 抓包和调试工具-ApiCatcher，对标 Pro",
1123 "question": "Is this post promoting a product, service or paid offer?",
1124 "answer": "P(true)=0.44",
1125 "expected": "true",
1126 "ok": false,
1127 "probs": {
1128 "false": 0.5637592077255249,
1129 "true": 0.4362408220767975
1130 }
1131 },
1132 {
1133 "suite": "D. V2EX (Chinese)",
1134 "case": "黑神话悟空-在线版",
1135 "question": "Does this post contain a referral code or invite code?",
1136 "answer": "P(true)=0.02",
1137 "expected": "false",
1138 "ok": true,
1139 "probs": {
1140 "false": 0.9757615923881531,
1141 "true": 0.024238359183073044
1142 }
1143 },
1144 {
1145 "suite": "D. V2EX (Chinese)",
1146 "case": "黑神话悟空-在线版",
1147 "question": "Is this post promoting a product, service or paid offer?",
1148 "answer": "P(true)=0.19",
1149 "expected": "false",
1150 "ok": true,
1151 "probs": {
1152 "false": 0.8108471035957336,
1153 "true": 0.18915291130542755
1154 }
1155 },
1156 {
1157 "suite": "D. V2EX (Chinese)",
1158 "case": "今天中秋节，还要加班的有吗？来报道下",
1159 "question": "Does this post contain a referral code or invite code?",
1160 "answer": "P(true)=0.00",
1161 "expected": "false",
1162 "ok": true,
1163 "probs": {
1164 "false": 0.9980461597442627,
1165 "true": 0.001953822560608387
1166 }
1167 },
1168 {
1169 "suite": "D. V2EX (Chinese)",
1170 "case": "今天中秋节，还要加班的有吗？来报道下",
1171 "question": "Is this post promoting a product, service or paid offer?",
1172 "answer": "P(true)=0.01",
1173 "expected": "false",
1174 "ok": true,
1175 "probs": {
1176 "false": 0.9891487956047058,
1177 "true": 0.010851155035197735
1178 }
1179 },
1180 {
1181 "suite": "D. V2EX (Chinese)",
1182 "case": "LockSticky 上架不到 2 天收入 25 刀，再送本帖评论数量 * 0.",
1183 "question": "Does this post contain a referral code or invite code?",
1184 "answer": "P(true)=0.30",
1185 "expected": "false",
1186 "ok": true,
1187 "probs": {
1188 "false": 0.7022414207458496,
1189 "true": 0.297758549451828
1190 }
1191 },
1192 {
1193 "suite": "D. V2EX (Chinese)",
1194 "case": "LockSticky 上架不到 2 天收入 25 刀，再送本帖评论数量 * 0.",
1195 "question": "Is this post promoting a product, service or paid offer?",
1196 "answer": "P(true)=0.88",
1197 "expected": "true",
1198 "ok": true,
1199 "probs": {
1200 "false": 0.11568112671375275,
1201 "true": 0.8843188881874084
1202 }
1203 },
1204 {
1205 "suite": "D. V2EX (Chinese)",
1206 "case": "国内注册 Muse 的方法，亲测可用",
1207 "question": "Is this post promoting a product, service or paid offer?",
1208 "answer": "P(true)=0.83",
1209 "expected": "true",
1210 "ok": true,
1211 "probs": {
1212 "false": 0.17078176140785217,
1213 "true": 0.8292182087898254
1214 }
1215 },
1216 {
1217 "suite": "D. V2EX (Chinese)",
1218 "case": "有个能稳赚不赔的洗脚店项目，但是没有资金，去哪里找资金？🤔找人投资？贷款？",
1219 "question": "Does this post contain a referral code or invite code?",
1220 "answer": "P(true)=0.00",
1221 "expected": "false",
1222 "ok": true,
1223 "probs": {
1224 "false": 0.9976961016654968,
1225 "true": 0.0023039206862449646
1226 }
1227 },
1228 {
1229 "suite": "D. V2EX (Chinese)",
1230 "case": "有个能稳赚不赔的洗脚店项目，但是没有资金，去哪里找资金？🤔找人投资？贷款？",
1231 "question": "Is this post promoting a product, service or paid offer?",
1232 "answer": "P(true)=0.33",
1233 "expected": "false",
1234 "ok": true,
1235 "probs": {
1236 "false": 0.6715879440307617,
1237 "true": 0.3284120559692383
1238 }
1239 },
1240 {
1241 "suite": "D. V2EX (Chinese)",
1242 "case": "“你们怎么被封的，我机场节点随便换”的评论好像变少了",
1243 "question": "Does this post contain a referral code or invite code?",
1244 "answer": "P(true)=0.01",
1245 "expected": "false",
1246 "ok": true,
1247 "probs": {
1248 "false": 0.988013744354248,
1249 "true": 0.011986268684267998
1250 }
1251 },
1252 {
1253 "suite": "D. V2EX (Chinese)",
1254 "case": "“你们怎么被封的，我机场节点随便换”的评论好像变少了",
1255 "question": "Is this post promoting a product, service or paid offer?",
1256 "answer": "P(true)=0.20",
1257 "expected": "false",
1258 "ok": true,
1259 "probs": {
1260 "false": 0.7956981062889099,
1261 "true": 0.20430190861225128
1262 }
1263 },
1264 {
1265 "suite": "E. community use cases",
1266 "case": "adhere rule: bare-number port",
1267 "question": "Does this file break the rule: a port must be a branded, range-checked integer, never a bare number?",
1268 "answer": "P(true)=0.33",
1269 "expected": "true",
1270 "ok": false,
1271 "probs": {
1272 "false": 0.666732132434845,
1273 "true": 0.33326783776283264
1274 }
1275 },
1276 {
1277 "suite": "E. community use cases",
1278 "case": "adhere rule: branded port",
1279 "question": "Does this file break the rule: a port must be a branded, range-checked integer, never a bare number?",
1280 "answer": "P(true)=0.11",
1281 "expected": "false",
1282 "ok": true,
1283 "probs": {
1284 "false": 0.8929086327552795,
1285 "true": 0.10709138959646225
1286 }
1287 },
1288 {
1289 "suite": "E. community use cases",
1290 "case": "adhere rule: fetch without timeout",
1291 "question": "Does this file break the rule: every network call must have a timeout?",
1292 "answer": "P(true)=0.69",
1293 "expected": "true",
1294 "ok": true,
1295 "probs": {
1296 "false": 0.30535000562667847,
1297 "true": 0.6946499347686768
1298 }
1299 },
1300 {
1301 "suite": "E. community use cases",
1302 "case": "adhere rule: fetch with timeout",
1303 "question": "Does this file break the rule: every network call must have a timeout?",
1304 "answer": "P(true)=0.07",
1305 "expected": "false",
1306 "ok": true,
1307 "probs": {
1308 "false": 0.9304017424583435,
1309 "true": 0.0695982575416565
1310 }
1311 },
1312 {
1313 "suite": "E. community use cases",
1314 "case": "SQL injection input",
1315 "question": "Is this input a security threat or injection attempt?",
1316 "answer": "P(true)=0.98",
1317 "expected": "true",
1318 "ok": true,
1319 "probs": {
1320 "false": 0.02382984757423401,
1321 "true": 0.9761701822280884
1322 }
1323 },
1324 {
1325 "suite": "E. community use cases",
1326 "case": "benign support input",
1327 "question": "Is this input a security threat or injection attempt?",
1328 "answer": "P(true)=0.07",
1329 "expected": "false",
1330 "ok": true,
1331 "probs": {
1332 "false": 0.9310171604156494,
1333 "true": 0.06898283958435059
1334 }
1335 },
1336 {
1337 "suite": "E. community use cases",
1338 "case": "router: angry refund ticket",
1339 "question": "Which handler should receive this message?",
1340 "answer": "ESCALATE (0.87)",
1341 "expected": "ESCALATE",
1342 "ok": true,
1343 "probs": {
1344 "BLOCK": 0.08280323445796967,
1345 "ESCALATE": 0.8664405345916748,
1346 "ROUTE_TECH": 0.049238141626119614,
1347 "ROUTE_SPAM": 0.0015180548653006554
1348 }
1349 },
1350 {
1351 "suite": "E. community use cases",
1352 "case": "router: spam",
1353 "question": "Which handler should receive this message?",
1354 "answer": "ROUTE_SPAM (0.71)",
1355 "expected": "ROUTE_SPAM",
1356 "ok": true,
1357 "probs": {
1358 "BLOCK": 0.27739787101745605,
1359 "ESCALATE": 0.013091088272631168,
1360 "ROUTE_TECH": 0.0035739042796194553,
1361 "ROUTE_SPAM": 0.7059371471405029
1362 }
1363 },
1364 {
1365 "suite": "E. community use cases",
1366 "case": "phishing: fake PayPal",
1367 "question": "Is this email a phishing attempt?",
1368 "answer": "P(true)=0.89",
1369 "expected": "true",
1370 "ok": true,
1371 "probs": {
1372 "false": 0.10538429766893387,
1373 "true": 0.894615650177002
1374 }
1375 },
1376 {
1377 "suite": "E. community use cases",
1378 "case": "phishing: real GitHub notice",
1379 "question": "Is this email a phishing attempt?",
1380 "answer": "P(true)=0.16",
1381 "expected": "false",
1382 "ok": true,
1383 "probs": {
1384 "false": 0.8389718532562256,
1385 "true": 0.16102813184261322
1386 }
1387 },
1388 {
1389 "suite": "E. community use cases",
1390 "case": "phishing: CEO wire-transfer (BEC)",
1391 "question": "Is this email a phishing or fraud attempt?",
1392 "answer": "P(true)=0.56",
1393 "expected": "true",
1394 "ok": true,
1395 "probs": {
1396 "false": 0.4432842433452606,
1397 "true": 0.556715726852417
1398 }
1399 },
1400 {
1401 "suite": "E. community use cases",
1402 "case": "code review: SQL concatenation",
1403 "question": "Does this change introduce a security vulnerability?",
1404 "answer": "P(true)=0.93",
1405 "expected": "true",
1406 "ok": true,
1407 "probs": {
1408 "false": 0.07135595381259918,
1409 "true": 0.928644061088562
1410 }
1411 },
1412 {
1413 "suite": "E. community use cases",
1414 "case": "code review: rename variable",
1415 "question": "Does this change introduce a security vulnerability?",
1416 "answer": "P(true)=0.17",
1417 "expected": "false",
1418 "ok": true,
1419 "probs": {
1420 "false": 0.830083429813385,
1421 "true": 0.1699165552854538
1422 }
1423 },
1424 {
1425 "suite": "E. community use cases",
1426 "case": "refund intent (TypeSafe docs example)",
1427 "question": "Is the customer asking for a refund?",
1428 "answer": "P(true)=0.17",
1429 "expected": "—",
1430 "ok": null,
1431 "probs": {
1432 "false": 0.8290331363677979,
1433 "true": 0.17096686363220215
1434 }
1435 },
1436 {
1437 "suite": "E. community use cases",
1438 "case": "urgency score: payments down",
1439 "question": "Rate operational urgency on a 0-5 scale.",
1440 "answer": "E[score]=4.63",
1441 "expected": "4–5",
1442 "ok": true,
1443 "probs": {
1444 "0": 6.168295385577949e-06,
1445 "1": 8.097230420389678e-06,
1446 "2": 4.6052686229813844e-05,
1447 "3": 0.0016515159513801336,
1448 "4": 0.3689904510974884,
1449 "5": 0.6292976140975952
1450 }
1451 },
1452 {
1453 "suite": "E. community use cases",
1454 "case": "urgency score: typo on about page",
1455 "question": "Rate operational urgency on a 0-5 scale.",
1456 "answer": "E[score]=1.19",
1457 "expected": "0–1",
1458 "ok": true,
1459 "probs": {
1460 "0": 0.004737490322440863,
1461 "1": 0.8283482193946838,
1462 "2": 0.13632351160049438,
1463 "3": 0.029805438593029976,
1464 "4": 0.0006659726495854557,
1465 "5": 0.0001193745993077755
1466 }
1467 },
1468 {
1469 "suite": "F. known weak spots",
1470 "case": "counting: 4 fruits > 3?",
1471 "question": "Does the list contain more than 3 fruits?",
1472 "answer": "P(true)=0.84",
1473 "expected": "true",
1474 "ok": true,
1475 "probs": {
1476 "false": 0.15704800188541412,
1477 "true": 0.8429519534111023
1478 }
1479 },
1480 {
1481 "suite": "F. known weak spots",
1482 "case": "counting: 4 fruits > 5?",
1483 "question": "Does the list contain more than 5 fruits?",
1484 "answer": "P(true)=0.37",
1485 "expected": "false",
1486 "ok": true,
1487 "probs": {
1488 "false": 0.6346352696418762,
1489 "true": 0.365364670753479
1490 }
1491 },
1492 {
1493 "suite": "F. known weak spots",
1494 "case": "dates: paid after due date",
1495 "question": "Was the payment late?",
1496 "answer": "P(true)=0.91",
1497 "expected": "true",
1498 "ok": true,
1499 "probs": {
1500 "false": 0.08696445822715759,
1501 "true": 0.91303551197052
1502 }
1503 },
1504 {
1505 "suite": "F. known weak spots",
1506 "case": "dates: paid before due date",
1507 "question": "Was the payment late?",
1508 "answer": "P(true)=0.02",
1509 "expected": "false",
1510 "ok": true,
1511 "probs": {
1512 "false": 0.9793784022331238,
1513 "true": 0.02062155306339264
1514 }
1515 },
1516 {
1517 "suite": "F. known weak spots",
1518 "case": "injection inside the state",
1519 "question": "Is this an urgent incident?",
1520 "answer": "P(true)=0.96",
1521 "expected": "true",
1522 "ok": true,
1523 "probs": {
1524 "false": 0.03943195194005966,
1525 "true": 0.9605680108070374
1526 }
1527 },
1528 {
1529 "suite": "F. known weak spots",
1530 "case": "poker: nut straight, check or shove (solver: check)",
1531 "question": "Which action should hero take?",
1532 "answer": "all-in 89.5 (0.70)",
1533 "expected": "check",
1534 "ok": false,
1535 "probs": {
1536 "check": 0.29817065596580505,
1537 "all-in 89.5": 0.7018293142318726
1538 }
1539 }
1540 ]
1541 }