eval/summarize.py
4.5 KB · 95 lines · python Raw
1 """Aggregate the per-case results of eval/run_test_case.py into a markdown table, optionally
2 paired against the reference numbers shipped in eval/cases/<task>_reference_metrics.json.
3
4 python eval/summarize.py --task rope [--root DIR] [--reference]
5 python eval/summarize.py --task rope --split val --root runs/<run>/val_epoch_014 \
6 --ref-root runs/nano_action_causal_rope_v1/val_epoch_014 # paired vs another run on the same val cases
7
8 short = 8 observed -> 8 predicted frames (metrics on the 8 predicted frames);
9 long = mean over the 15 autoregressive windows (120 generated frames).
10 """
11 from __future__ import annotations
12
13 import argparse
14 import json
15 import os
16 from pathlib import Path
17
18 REPO = Path(__file__).resolve().parents[1]
19 STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
20 METRICS = ["psnr", "ssim", "lpips"]
21
22
23 def load(root: Path, man: dict) -> dict:
24 out = {}
25 for c in man["cases"]:
26 d = root / c["output_subdir"]
27 if not (d / "complete.json").is_file():
28 continue
29 w = json.loads((d / "window_eval.json").read_text())["results"][0]
30 r = {"category": c["category"], "short": {s: {m: w["image"][s][m] for m in METRICS} for s in STREAMS}}
31 rr = d / "reference_rollout.json"
32 if c["rollout_windows"] and rr.is_file():
33 wins = json.loads(rr.read_text())[0]["windows"]
34 r["long_mean"] = {s: {m: sum(x["image"][s][m] for x in wins) / len(wins) for m in METRICS} for s in STREAMS}
35 out[c["output_subdir"]] = r
36 return out
37
38
39 def mean(xs):
40 return sum(xs) / len(xs) if xs else float("nan")
41
42
43 def table(res: dict, hz: str, title: str) -> list[str]:
44 rows = [r for r in res.values() if hz in r]
45 lines = [f"### {title} (n={len(rows)})", "", "| stream | PSNR ↑ | SSIM ↑ | LPIPS ↓ |", "|---|---:|---:|---:|"]
46 for s in STREAMS:
47 v = [mean([r[hz][s][m] for r in rows]) for m in METRICS]
48 lines.append(f"| {s} | {v[0]:.2f} | {v[1]:.4f} | {v[2]:.4f} |")
49 return lines + [""]
50
51
52 def main():
53 ap = argparse.ArgumentParser()
54 tasks = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
55 ap.add_argument("--task", required=True, choices=tasks)
56 ap.add_argument("--root", default=None, help="default: $ACTS_RUNS_ROOT/<run_name>/test_<ckpt stem of the manifest>")
57 ap.add_argument("--split", default="test", choices=["test", "val"])
58 ap.add_argument("--reference", action="store_true",
59 help="also print paired deltas vs the shipped reference metrics (test split only)")
60 ap.add_argument("--ref-root", default=None,
61 help="also print paired deltas vs another result dir on the same cases (e.g. the parent run's "
62 "val_epoch_014); use this for the val split")
63 a = ap.parse_args()
64 man = json.loads((REPO / "eval" / "cases" / f"{a.task}_{a.split}_cases.json").read_text())
65 runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
66 root = Path(a.root or runs / man["run_name"] / f"{a.split}_{Path(man['checkpoint']).stem}")
67 res = load(root, man)
68 print(f"# {root.parent.name} {root.name} ({len(res)}/{len(man['cases'])} cases complete) -- {root}\n")
69 print("\n".join(table(res, "short", "short horizon: 8 -> 8 frames")))
70 print("\n".join(table(res, "long_mean", "long horizon: mean over 15 chained windows")))
71 refs = []
72 if a.reference:
73 if a.split != "test":
74 raise SystemExit("--reference is the shipped TEST reference; for val use --ref-root")
75 ref_file = REPO / "eval" / "cases" / f"{a.task}_reference_metrics.json"
76 if not ref_file.is_file():
77 raise SystemExit(f"no reference metrics for {a.task}: {ref_file}")
78 refs.append(("reference", {r["output_subdir"]: r for r in json.loads(ref_file.read_text())["cases"]}))
79 if a.ref_root:
80 rr = Path(a.ref_root)
81 refs.append((f"{rr.parent.name}/{rr.name}", load(rr, man)))
82 for name, ref in refs:
83 print(f"### paired difference vs {name} (this run − {name}), PSNR dB, n = {len(ref)} reference cases\n")
84 print("| horizon | stream | n | mean Δ | max |Δ| |\n|---|---|---:|---:|---:|")
85 for hz in ("short", "long_mean"):
86 for s in STREAMS:
87 d = [res[k][hz][s]["psnr"] - ref[k][hz][s]["psnr"] for k in res if hz in res[k] and hz in ref.get(k, {})]
88 if d:
89 print(f"| {hz} | {s} | {len(d)} | {mean(d):+.3f} | {max(abs(x) for x in d):.3f} |")
90 print()
91
92
93 if __name__ == "__main__":
94 main()
95