Code / tools/new_experiment.py

tools/new_experiment.py 149 lines
#!/usr/bin/env python3
# =============================================================================
#  Project   : localvm-research
#  File      : tools/new_experiment.py
#  Purpose   : Scaffold a header-compliant experiment directory (CLAUDE.md §3, §10)
#  Author    : Simon-Pierre Boucher
#  Contact   : contact@spboucher.ai
#  Created   : 2026-08-11
#  Modified  : 2026-08-11
#  Platform  : macOS / Apple Silicon (arm64)
#  License   : All rights reserved (research code)
# =============================================================================
"""Scaffold a new experiment directory with the mandatory structure.

Creates: README.md, hypothesis.md (seven-field scientific block), benchmark.py,
implementation/, results/, analysis.md — all with conforming author headers.

Usage:
    python3 tools/new_experiment.py experiments/micro/expX_name "One-line purpose"
    python3 tools/new_experiment.py experiments/candidate_04 "Candidate: ..."
"""

from __future__ import annotations

import sys
from datetime import date
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parent.parent
TODAY = date.today().isoformat()

PY_HEADER = """\
# =============================================================================
#  Project   : localvm-research
#  File      : {rel}
#  Purpose   : {purpose}
#  Author    : Simon-Pierre Boucher
#  Contact   : contact@spboucher.ai
#  Created   : {today}
#  Modified  : {today}
#  Platform  : macOS / Apple Silicon (arm64)
#  License   : All rights reserved (research code)
# =============================================================================
"""

MD_HEADER = """\
---
project: localvm-research
document: {doc}
author: Simon-Pierre Boucher
contact: contact@spboucher.ai
created: {today}
status: draft
---
"""

HYPOTHESIS_BODY = """
# Hypothesis — {name}

```text
Hypothesis
  <what we believe and why>

Falsification criterion
  <the concrete measurable outcome that would prove this wrong>

Method
  <exact procedure, model(s), data, seeds, measurement points>

Baseline
  <what this is compared against — no straw men>

Result
  <filled after the run: numbers, with mean/median/std and run count>

Interpretation
  <what the numbers mean; alternative explanations considered>

Next experiment
  <the most informative follow-up given this result>
```
"""

BENCHMARK_BODY = '''
"""Benchmark entry point for {name}.

Must embed the hardware manifest in all result output
(see benchmarks/hardware_manifest.py) and write results to
results/{name}/<timestamp>/.
"""

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[{depth}] / "benchmarks"))
from hardware_manifest import collect_manifest  # noqa: E402


def main() -> None:
    manifest = collect_manifest()
    raise NotImplementedError("experiment not yet implemented")


if __name__ == "__main__":
    main()
'''


def scaffold(exp_dir: Path, purpose: str) -> None:
    if exp_dir.exists() and any(exp_dir.iterdir()):
        sys.exit(f"error: {exp_dir} already exists and is not empty")
    name = exp_dir.name
    rel = exp_dir.relative_to(REPO_ROOT)
    (exp_dir / "implementation").mkdir(parents=True, exist_ok=True)
    (exp_dir / "results").mkdir(exist_ok=True)

    def md(doc: str) -> str:
        return MD_HEADER.format(doc=doc, today=TODAY)

    (exp_dir / "README.md").write_text(
        md(f"{name}/README") + f"\n# {name}\n\n{purpose}\n\nStatus: scaffolded {TODAY}, not yet run.\n"
    )
    (exp_dir / "hypothesis.md").write_text(
        md(f"{name}/hypothesis") + HYPOTHESIS_BODY.format(name=name)
    )
    (exp_dir / "analysis.md").write_text(
        md(f"{name}/analysis") + f"\n# Analysis — {name}\n\n*To be written after results exist. "
        "Must include the seven-field block and the evidence standard of CLAUDE.md §10.*\n"
    )
    depth = len(rel.parts)  # parents[] index up to repo root
    (exp_dir / "benchmark.py").write_text(
        PY_HEADER.format(rel=rel / "benchmark.py", purpose=f"Benchmark runner: {purpose}", today=TODAY)
        + BENCHMARK_BODY.format(name=name, depth=depth)
    )
    print(f"scaffolded {rel} ({purpose})")


def main() -> None:
    if len(sys.argv) < 3:
        sys.exit(__doc__)
    exp_dir = (REPO_ROOT / sys.argv[1]).resolve()
    if REPO_ROOT not in exp_dir.parents:
        sys.exit("error: experiment directory must live inside the repository")
    scaffold(exp_dir, sys.argv[2])


if __name__ == "__main__":
    main()