benchmark.py

"""
benchmark.py v4 — collect_data.py와 동일한 파이프라인
baseline: src → clang-18 -O1 → bc → exe
AI:       src → clang-18 -O1 → bc → opt → bc → exe
ffast:    src → clang-18 -O1 -ffast-math → ff_bc → opt → bc → exe
"""
import subprocess, time, os, json
import numpy as np
import glob

os.chdir(os.path.expanduser('~/projects/machineai'))
CLANG, OPT, REPEAT = "clang-18", "opt-18", 9

from passes_config import PASSES_14 as PASSES

with open("data/claude_analysis.json") as f:
    analyses = json.load(f)
action_map = {a["file"]: a["analysis"]["actual_best_action"] for a in analyses}

SOURCE_FILES = sorted(glob.glob("benchmarks/*.c"))

def measure(exe, n=REPEAT):
    times = []
    for _ in range(n):
        t0 = time.perf_counter()
        r = subprocess.run([exe], capture_output=True)
        t1 = time.perf_counter()
        if r.returncode == 0:
            times.append(t1 - t0)
    return float(np.median(times)) if times else 999.0

def bc_to_exe(bc, exe):
    subprocess.run([CLANG, "-O1", bc, "-o", exe, "-lm"], capture_output=True)

def compile_and_measure(src, action):
    p = PASSES[action]
    base_bc  = src.replace(".c", ".bc")
    base_exe = src.replace(".c", "_base_exe")
    ai_exe   = src.replace(".c", "_ai_exe")

    # 베이스라인: -O1 bc → exe
    subprocess.run([CLANG, "-O1", "-emit-llvm", "-c", src, "-o", base_bc], capture_output=True)
    bc_to_exe(base_bc, base_exe)

    if p.startswith("__ffast__"):
        ff_bc  = src.replace(".c", "_ff.bc")
        out_bc = src.replace(".c", "_ai.bc")
        passes_str = p[len("__ffast__"):]
        subprocess.run([CLANG, "-O1", "-ffast-math", "-emit-llvm", "-c", src, "-o", ff_bc], capture_output=True)
        if passes_str:
            subprocess.run([OPT, f"--passes={passes_str}", ff_bc, "-o", out_bc], capture_output=True)
            bc_to_exe(out_bc, ai_exe)
        else:
            bc_to_exe(ff_bc, ai_exe)
    else:
        out_bc = src.replace(".c", "_ai.bc")
        if p:
            subprocess.run([OPT, f"--passes={p}", base_bc, "-o", out_bc], capture_output=True)
            bc_to_exe(out_bc, ai_exe)
        else:
            bc_to_exe(base_bc, ai_exe)

    t_base = measure(base_exe)
    t_ai   = measure(ai_exe)
    pct    = (t_base - t_ai) / (t_base + 1e-9) * 100
    ff     = "[ff]" if p.startswith("__ffast__") else "    "
    return t_base, t_ai, pct, ff

print("=" * 64)
print("  BENCHMARK v4 — collect_data 동일 파이프라인 (action 0~13)")
print("=" * 64)

total_pct = []
for src in SOURCE_FILES:
    name = os.path.basename(src)
    if name not in action_map:
        continue
    action = action_map[name]
    t_base, t_ai, pct, ff = compile_and_measure(src, action)
    total_pct.append(pct)
    print(f"  {name:<22} act={action:2d}{ff}  base={t_base*1000:7.1f}ms  ai={t_ai*1000:7.1f}ms  {pct:+.1f}%")

print("=" * 64)
avg = np.mean(total_pct) if total_pct else 0
print(f"  평균 speedup: {avg:+.2f}%  ({len(total_pct)}개 파일)")
print("=" * 64)

THRESHOLD = 52.7
if avg >= THRESHOLD:
    print(f"\n  ✅ {avg:.1f}% ≥ {THRESHOLD}% → git commit 조건 충족!")
else:
    print(f"\n  ⏳ {avg:.1f}% < {THRESHOLD}% → commit 기준 미달 (목표: +{THRESHOLD}%)")