eval/run_test_case.py
8.3 KB · 150 lines · python Raw
1 """Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render
2 GT/prediction videos for the five image streams.
3
4 python eval/run_test_case.py --task rope --case 0 # one case
5 python eval/run_test_case.py --task rope --case all # every case, sequentially
6 python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt
7 python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt
8
9 The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the
10 exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py).
11 Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json,
12 strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py.
13 Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT)."""
14 from __future__ import annotations
15
16 import argparse
17 import json
18 import os
19 import subprocess
20 import sys
21 from pathlib import Path
22
23 import imageio.v2 as imageio
24 import numpy as np
25 from PIL import Image, ImageDraw
26
27 REPO = Path(__file__).resolve().parents[1]
28 STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"]
29 TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json"))
30 # data tree from the split file of the case manifest
31 TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope",
32 "mixed_pusht_wrist_holdout": "mixed_pushT"}
33
34
35 def default_out_root(ckpt: Path, split: str = "test") -> Path:
36 """$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014"""
37 runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs"))
38 return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}"
39
40
41 def case_file(task: str, split: str) -> Path:
42 f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json"
43 if not f.is_file():
44 raise SystemExit(f"no {split} cases for task {task}: {f}")
45 return f
46
47
48 def env(name: str) -> str:
49 v = os.environ.get(name)
50 if not v:
51 raise SystemExit(f"${name} is not set -- run `source env.sh` first")
52 return v
53
54
55 def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool):
56 out = out_root / case["output_subdir"]
57 if skip_done and (out / "complete.json").is_file():
58 print("[skip] done:", out, flush=True)
59 return
60 out.mkdir(parents=True, exist_ok=True)
61 (out / "case.json").write_text(json.dumps(case, indent=2))
62 print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True)
63 data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT")
64 tree = TREE_OF_SPLIT[Path(man["split_file"]).stem]
65 overrides = [
66 f"data.latent_root={data}/{tree}_latents",
67 f"data.action_root={data}/{tree}_actions",
68 f"data.split_file={REPO / man['split_file']}",
69 f"model.nano_checkpoint={ckroot}/Cosmos3-Nano",
70 f"out_dir={out_root}",
71 ]
72 subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout",
73 "--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1",
74 "--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]),
75 "--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)),
76 "--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO)
77
78 w = json.loads((out / "window_eval.json").read_text())["results"][0]
79 if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])):
80 raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, "
81 f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)")
82
83 # Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once,
84 # then append each rollout window's 8 future frames.
85 stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}"
86 if case["rollout_windows"]:
87 rolls = json.loads((out / "reference_rollout.json").read_text())
88 assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout"
89 assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early"
90 for mode in (("short", "long") if case["rollout_windows"] else ("short",)):
91 video_path = out / f"{mode}_gt_pred.mp4"
92 count = 0
93 with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer:
94 for k in range(1 if mode == "short" else case["rollout_windows"]):
95 suffix = "" if mode == "short" else f"_roll{k}"
96 strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS]
97 h = (strips[0].shape[0] - 4) // 2
98 wd = strips[0].shape[1] // 16 - 2
99 for t in range(0 if k == 0 else 8, 16):
100 canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black")
101 draw = ImageDraw.Draw(canvas)
102 for col, (stream, strip) in enumerate(zip(STREAMS, strips)):
103 x = t * (wd + 2)
104 canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32))
105 canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64))
106 draw.text((col * wd + 4, 16), "GT: " + stream, fill="white")
107 draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white")
108 label = "observed history" if count < 8 else "predicted future"
109 draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white")
110 writer.append_data(np.asarray(canvas))
111 count += 1
112 print("[video]", video_path, "frames=", count, flush=True)
113 (out / "complete.json").write_text(json.dumps(dict(
114 short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0,
115 rollout_windows=case["rollout_windows"])))
116 print("[case done]", out, flush=True)
117
118
119 def main():
120 ap = argparse.ArgumentParser()
121 ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)")
122 ap.add_argument("--split", default="test", choices=["test", "val"],
123 help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)")
124 ap.add_argument("--case", required=True, help="case number, comma list, or 'all'")
125 ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases")
126 ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>")
127 ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)")
128 ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)")
129 ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json")
130 a = ap.parse_args()
131
132 man = json.loads(case_file(a.task, a.split).read_text())
133 assert man["split"] == a.split, (man["split"], a.split)
134 ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"])
135 if not ckpt.is_file():
136 raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)")
137 out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split)
138 out_root.mkdir(parents=True, exist_ok=True)
139 ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")]
140 steps = a.steps or man.get("denoising_steps_per_latent_block", 35)
141 for i in ids:
142 case = dict(man["cases"][i])
143 if a.short_only:
144 case["rollout_windows"] = 0
145 run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo)
146
147
148 if __name__ == "__main__":
149 main()
150