eval/run_test_case.py
| 1 | """Run one fixed test case (short 8->8 window + 15-window autoregressive rollout) and render |
| 2 | GT/prediction videos for the five image streams. |
| 3 | |
| 4 | python eval/run_test_case.py --task rope --case 0 # one case |
| 5 | python eval/run_test_case.py --task rope --case all # every case, sequentially |
| 6 | python eval/run_test_case.py --task rope --case all --ckpt runs/nano_action_causal_rope_sensoronly_v1/epoch_019.pt |
| 7 | python eval/run_test_case.py --task rope --split val --case all --ckpt checkpoints/nano_action_causal_rope_v1/epoch_014.pt |
| 8 | |
| 9 | The case list (episode, dataset index, contact class) is eval/cases/<task>_<split>_cases.json (split = test, the |
| 10 | exact set used for the reference test numbers, or val for pushT / rope, built by scripts/build_val_cases.py). |
| 11 | Outputs go to $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>/<output_subdir>/ (window_eval.json, reference_rollout.json, |
| 12 | strip PNGs, short_gt_pred.mp4, long_gt_pred.mp4). Summarise with eval/summarize.py. |
| 13 | Needs env.sh sourced (ACTS_DATA_ROOT, ACTS_CKPT_ROOT, VTM_SOURCE_VIDEO_ROOT).""" |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import argparse |
| 17 | import json |
| 18 | import os |
| 19 | import subprocess |
| 20 | import sys |
| 21 | from pathlib import Path |
| 22 | |
| 23 | import imageio.v2 as imageio |
| 24 | import numpy as np |
| 25 | from PIL import Image, ImageDraw |
| 26 | |
| 27 | REPO = Path(__file__).resolve().parents[1] |
| 28 | STREAMS = ["video", "wrist_left", "wrist_right", "tactile_left", "tactile_right"] |
| 29 | TASKS = sorted(f.name[:-len("_test_cases.json")] for f in (REPO / "eval" / "cases").glob("*_test_cases.json")) |
| 30 | # data tree from the split file of the case manifest |
| 31 | TREE_OF_SPLIT = {"mixed_mb_wrist_holdout": "mixed_mb", "mixed_rope_wrist_holdout": "mixed_rope", |
| 32 | "mixed_pusht_wrist_holdout": "mixed_pushT"} |
| 33 | |
| 34 | |
| 35 | def default_out_root(ckpt: Path, split: str = "test") -> Path: |
| 36 | """$ACTS_RUNS_ROOT/<run dir of the ckpt>/<split>_<ckpt stem>, e.g. .../nano_action_causal_rope_v1/val_epoch_014""" |
| 37 | runs = Path(os.environ.get("ACTS_RUNS_ROOT", REPO / "runs")) |
| 38 | return runs / ckpt.absolute().parent.name / f"{split}_{ckpt.stem}" |
| 39 | |
| 40 | |
| 41 | def case_file(task: str, split: str) -> Path: |
| 42 | f = REPO / "eval" / "cases" / f"{task}_{split}_cases.json" |
| 43 | if not f.is_file(): |
| 44 | raise SystemExit(f"no {split} cases for task {task}: {f}") |
| 45 | return f |
| 46 | |
| 47 | |
| 48 | def env(name: str) -> str: |
| 49 | v = os.environ.get(name) |
| 50 | if not v: |
| 51 | raise SystemExit(f"${name} is not set -- run `source env.sh` first") |
| 52 | return v |
| 53 | |
| 54 | |
| 55 | def run_case(man: dict, case: dict, ckpt: Path, out_root: Path, steps: int, skip_done: bool): |
| 56 | out = out_root / case["output_subdir"] |
| 57 | if skip_done and (out / "complete.json").is_file(): |
| 58 | print("[skip] done:", out, flush=True) |
| 59 | return |
| 60 | out.mkdir(parents=True, exist_ok=True) |
| 61 | (out / "case.json").write_text(json.dumps(case, indent=2)) |
| 62 | print("[case]", json.dumps({k: case[k] for k in ("task", "episode", "category", "index", "t_start", "rollout_windows")}), flush=True) |
| 63 | data, ckroot = env("ACTS_DATA_ROOT"), env("ACTS_CKPT_ROOT") |
| 64 | tree = TREE_OF_SPLIT[Path(man["split_file"]).stem] |
| 65 | overrides = [ |
| 66 | f"data.latent_root={data}/{tree}_latents", |
| 67 | f"data.action_root={data}/{tree}_actions", |
| 68 | f"data.split_file={REPO / man['split_file']}", |
| 69 | f"model.nano_checkpoint={ckroot}/Cosmos3-Nano", |
| 70 | f"out_dir={out_root}", |
| 71 | ] |
| 72 | subprocess.run([sys.executable, "-u", "-m", "vtm_cosmos.infer.rollout", |
| 73 | "--ckpt", str(ckpt), "--split", man["split"], "--window-stride", "1", |
| 74 | "--windows", str(case["index"]), "--rollout-windows", str(case["rollout_windows"]), |
| 75 | "--steps", str(steps), "--solver", man.get("solver", "unipc"), "--seed", str(man.get("seed", 0)), |
| 76 | "--out-dir", str(out), "--override", *overrides], check=True, cwd=REPO) |
| 77 | |
| 78 | w = json.loads((out / "window_eval.json").read_text())["results"][0] |
| 79 | if (w["episode"], int(w["t_start"])) != (case["episode"], int(case["t_start"])): |
| 80 | raise RuntimeError(f"dataset index {case['index']} resolved to {w['episode']}@{w['t_start']}, " |
| 81 | f"expected {case['episode']}@{case['t_start']} (wrong split / data version?)") |
| 82 | |
| 83 | # Each saved strip holds 16 frames, GT on top and prediction below. Show the 8 observed frames once, |
| 84 | # then append each rollout window's 8 future frames. |
| 85 | stem = f"{Path(case['episode']).stem}_t{case['t_start']:05d}" |
| 86 | if case["rollout_windows"]: |
| 87 | rolls = json.loads((out / "reference_rollout.json").read_text()) |
| 88 | assert len(rolls) == 1 and len(rolls[0]["windows"]) == case["rollout_windows"], "incomplete rollout" |
| 89 | assert not any("stopped" in x for x in rolls[0]["windows"]), "rollout stopped early" |
| 90 | for mode in (("short", "long") if case["rollout_windows"] else ("short",)): |
| 91 | video_path = out / f"{mode}_gt_pred.mp4" |
| 92 | count = 0 |
| 93 | with imageio.get_writer(video_path, fps=15, codec="libx264", quality=8) as writer: |
| 94 | for k in range(1 if mode == "short" else case["rollout_windows"]): |
| 95 | suffix = "" if mode == "short" else f"_roll{k}" |
| 96 | strips = [imageio.imread(out / f"{stem}{suffix}_{s}.png") for s in STREAMS] |
| 97 | h = (strips[0].shape[0] - 4) // 2 |
| 98 | wd = strips[0].shape[1] // 16 - 2 |
| 99 | for t in range(0 if k == 0 else 8, 16): |
| 100 | canvas = Image.new("RGB", (len(STREAMS) * wd, 2 * h + 64), "black") |
| 101 | draw = ImageDraw.Draw(canvas) |
| 102 | for col, (stream, strip) in enumerate(zip(STREAMS, strips)): |
| 103 | x = t * (wd + 2) |
| 104 | canvas.paste(Image.fromarray(strip[:h, x:x + wd]), (col * wd, 32)) |
| 105 | canvas.paste(Image.fromarray(strip[h + 4:h + 4 + h, x:x + wd]), (col * wd, h + 64)) |
| 106 | draw.text((col * wd + 4, 16), "GT: " + stream, fill="white") |
| 107 | draw.text((col * wd + 4, h + 48), "PRED: " + stream, fill="white") |
| 108 | label = "observed history" if count < 8 else "predicted future" |
| 109 | draw.text((4, 2), f"{mode} | {case['category']} | frame {count} | {label}", fill="white") |
| 110 | writer.append_data(np.asarray(canvas)) |
| 111 | count += 1 |
| 112 | print("[video]", video_path, "frames=", count, flush=True) |
| 113 | (out / "complete.json").write_text(json.dumps(dict( |
| 114 | short_frames=16, long_frames=8 + 8 * case["rollout_windows"] if case["rollout_windows"] else 0, |
| 115 | rollout_windows=case["rollout_windows"]))) |
| 116 | print("[case done]", out, flush=True) |
| 117 | |
| 118 | |
| 119 | def main(): |
| 120 | ap = argparse.ArgumentParser() |
| 121 | ap.add_argument("--task", required=True, choices=TASKS, help="case set (eval/cases/<task>_<split>_cases.json)") |
| 122 | ap.add_argument("--split", default="test", choices=["test", "val"], |
| 123 | help="test (default) or val (pushT / rope only: eval/cases/<task>_val_cases.json)") |
| 124 | ap.add_argument("--case", required=True, help="case number, comma list, or 'all'") |
| 125 | ap.add_argument("--ckpt", default=None, help="default: $ACTS_CKPT_ROOT/<manifest checkpoint>; pass a TODO-run ckpt to evaluate it on the same cases") |
| 126 | ap.add_argument("--out-root", default=None, help="default: $ACTS_RUNS_ROOT/<ckpt run dir>/<split>_<ckpt stem>") |
| 127 | ap.add_argument("--steps", type=int, default=None, help="UniPC steps per latent block (default 35 = paper)") |
| 128 | ap.add_argument("--short-only", action="store_true", help="skip the 15-window rollout (~10x faster)") |
| 129 | ap.add_argument("--redo", action="store_true", help="re-run cases that already have complete.json") |
| 130 | a = ap.parse_args() |
| 131 | |
| 132 | man = json.loads(case_file(a.task, a.split).read_text()) |
| 133 | assert man["split"] == a.split, (man["split"], a.split) |
| 134 | ckpt = Path(a.ckpt or Path(env("ACTS_CKPT_ROOT")) / man["checkpoint"]) |
| 135 | if not ckpt.is_file(): |
| 136 | raise SystemExit(f"checkpoint not found: {ckpt} (python scripts/download.py --what ckpts)") |
| 137 | out_root = Path(a.out_root) if a.out_root else default_out_root(ckpt, a.split) |
| 138 | out_root.mkdir(parents=True, exist_ok=True) |
| 139 | ids = range(len(man["cases"])) if a.case == "all" else [int(x) for x in a.case.split(",")] |
| 140 | steps = a.steps or man.get("denoising_steps_per_latent_block", 35) |
| 141 | for i in ids: |
| 142 | case = dict(man["cases"][i]) |
| 143 | if a.short_only: |
| 144 | case["rollout_windows"] = 0 |
| 145 | run_case(man, case, ckpt, out_root, steps, skip_done=not a.redo) |
| 146 | |
| 147 | |
| 148 | if __name__ == "__main__": |
| 149 | main() |
| 150 | |