Building an evaluation harness

{"id": "s04", "slice": "credit", "must_pass": true, "order": {"SalesOrder": "9100004", "SoldToParty": "CUST-A", "TotalNetAmount": "51200.00", "DeliveryBlockReason": "01", "TotalCreditCheckStatus": "B", "...": "..."}}

"""Unit 8: an evaluation harness for the blocked-order triage writer.

A harness is the machinery around your tests: a fixed suite of cases, a runner, scorers, a results
store and a gate. Every run is recorded with what was tested (prompt, model, suite, Git commit),
compared with a saved baseline, and either passes or fails the gate.

How to run (from your course folder, with .venv turned on):
    python unit08/harness.py init                        # write the suite, two prompt versions, the config
    python unit08/harness.py run                         # evaluate the prompt named in eval_config.json
    python unit08/harness.py run --prompt v2             # evaluate another prompt version
    python unit08/harness.py history                     # every recorded run, oldest first
    python unit08/harness.py baseline                    # make the latest run the baseline
    python unit08/harness.py compare                     # latest run vs. baseline; exit code 1 if the gate fails
    python unit08/harness.py run --model anthropic/claude-opus-5-5   # a real model writes the notes (small charge)
    python unit08/harness.py run --judge anthropic/claude-opus-5-5   # add a model judge as a fifth scorer (small charge)

Without --model, a made-up "offline writer" stands in for the model, so every step works with no account.
"""
import argparse
import hashlib
import json
import os
import re
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path

from dotenv import load_dotenv

HERE = Path(__file__).resolve().parent
SUITE = HERE / "data" / "triage_suite.jsonl"
PROMPTS = HERE / "prompts"
CONFIG = HERE / "eval_config.json"
RESULTS = HERE / "results"
RUNS = RESULTS / "runs.jsonl"
BASELINE = RESULTS / "baseline.json"
REPORT = RESULTS / "report.md"
LOGS = HERE / "logs" / "harness"

# ---------------------------------------------------------------- the suite (made-up, SAP-shaped)


def order(number, customer, amount, delivery_block="", billing_block="", credit_status=""):
    """Header fields shaped like Unit 1's sales order. The codes are placeholders, not SAP meanings."""
    return {"SalesOrder": number, "SoldToParty": customer, "TotalNetAmount": amount,
            "TransactionCurrency": "USD", "TotalCreditCheckStatus": credit_status,
            "DeliveryBlockReason": delivery_block, "HeaderBillingBlockReason": billing_block}


SAMPLE_SUITE = [
    {"id": "s01", "slice": "credit", "must_pass": True, "order": order("9100001", "CUST-A", "72400.00", "01", "", "B")},
    {"id": "s02", "slice": "credit", "must_pass": False, "order": order("9100002", "CUST-B", "8300.00", "01", "", "B")},
    {"id": "s03", "slice": "credit", "must_pass": False, "order": order("9100003", "CUST-C", "15900.00", "01", "", "B")},
    {"id": "s04", "slice": "credit", "must_pass": True, "order": order("9100004", "CUST-A", "51200.00", "01", "", "B")},
    {"id": "s05", "slice": "billing", "must_pass": False, "order": order("9100005", "CUST-D", "940.00", "", "02")},
    {"id": "s06", "slice": "billing", "must_pass": False, "order": order("9100006", "CUST-E", "2210.00", "", "04")},
    {"id": "s07", "slice": "delivery", "must_pass": False, "order": order("9100007", "CUST-F", "3150.00", "03")},
    {"id": "s08", "slice": "delivery", "must_pass": False, "order": order("9100008", "CUST-G", "610.00", "05")},
    {"id": "s09", "slice": "multi", "must_pass": False, "order": order("9100009", "CUST-H", "4480.00", "03", "02")},
    {"id": "s10", "slice": "multi", "must_pass": False, "order": order("9100010", "CUST-B", "12650.00", "01", "02", "B")},
    {"id": "s11", "slice": "no-block", "must_pass": True, "order": order("9100011", "CUST-I", "560.00")},
    {"id": "s12", "slice": "no-block", "must_pass": True, "order": order("9100012", "CUST-J", "7020.00")},
]

PROMPT_V1 = """You write short triage notes for credit and order-management analysts about blocked sales orders.
Use only the order fields given. Do not guess what block or status codes mean; codes differ by system.
Mention every block or status that is set. If no block is set, say the data does not show why the order is held.
Reply with JSON only: {"likely_cause": "...", "next_check": "...", "confidence": "low|medium|high"}"""

PROMPT_V2 = """You write very short triage notes for analysts about blocked sales orders.
Focus on the main problem only. Keep each field under 12 words.
Reply with JSON only: {"likely_cause": "...", "next_check": "...", "confidence": "low|medium|high"}"""

SAMPLE_CONFIG = {
    "prompt": "v1",
    "gate": {
        "min": {"valid_format": 1.0, "covers_facts": 0.8, "no_code_guessing": 0.9, "no_outside_orders": 0.9},
        "max_drop": 0.05,
        "must_pass_cases": True,
    },
}


def sha(text: str) -> str:
    return hashlib.sha256(text.encode("utf-8")).hexdigest()[:10]


def read_suite() -> list:
    if not SUITE.exists():
        sys.exit("No suite yet. Run: python unit08/harness.py init")
    cases = [json.loads(line) for line in SUITE.read_text(encoding="utf-8").splitlines() if line.strip()]
    ids = [c["id"] for c in cases]
    if len(ids) != len(set(ids)):
        sys.exit("Two cases in the suite share an id. Every case needs its own id.")
    return cases


def read_config() -> dict:
    if not CONFIG.exists():
        sys.exit("No eval_config.json yet. Run: python unit08/harness.py init")
    return json.loads(CONFIG.read_text(encoding="utf-8"))


def read_prompt(version: str) -> str:
    path = PROMPTS / f"{version}.txt"
    if not path.exists():
        sys.exit(f"No prompt file {path.name} in unit08/prompts. Run init, or create the file.")
    return path.read_text(encoding="utf-8").strip()


def git_commit() -> str:
    """The current commit, plus '+changes' if files are edited but not committed."""
    try:
        commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True,
                                check=True).stdout.strip()
        dirty = subprocess.run(["git", "status", "--porcelain", "--", str(HERE)], capture_output=True,
                               text=True).stdout.strip()
        return commit + ("+changes" if dirty else "")
    except (OSError, subprocess.CalledProcessError):
        return "no-git"


# ---------------------------------------------------------------- the offline writer (stands in for a model)


def offline_note(version: str, fields: dict) -> str:
    """Made-up behavior for each prompt version, so the harness has something to measure without a model.
    v1 follows its rules, with one flaw: on CUST-B's order with two blocks it cites another order.
    v2 ('shorter notes') keeps only the first block and, on large credit orders, guesses what code 01 means."""
    delivery, billing = fields["DeliveryBlockReason"], fields["HeaderBillingBlockReason"]
    credit, amount = fields["TotalCreditCheckStatus"], float(fields["TotalNetAmount"])
    if version == "v2":
        if delivery and credit and amount > 50000:
            return json.dumps({"likely_cause": "Delivery block 01 means the credit limit is exceeded.",
                               "next_check": "Release the order in credit management.", "confidence": "high"})
        if delivery:
            return json.dumps({"likely_cause": "A delivery block is set.",
                               "next_check": "Ask sales about the delivery block.", "confidence": "medium"})
        if billing:
            return json.dumps({"likely_cause": "A billing block is set.",
                               "next_check": "Ask billing about the block.", "confidence": "medium"})
        return json.dumps({"likely_cause": "No block is set; the data does not show why it is held.",
                           "next_check": "Ask who flagged the order.", "confidence": "low"})
    parts, checks = [], []
    if delivery:
        parts.append(f"a delivery block ({delivery})")
        checks.append("ask sales which delivery block this code is in your system")
    if billing:
        parts.append(f"a billing block ({billing})")
        checks.append("ask billing what the block code stands for in your system")
    if credit:
        parts.append(f"a credit check status ({credit})")
        checks.insert(0, "check the customer's credit exposure and open items")
    if not parts:
        return json.dumps({"likely_cause": "No block or credit status is set, so the data does not show why the order is held.",
                           "next_check": "Confirm with the analyst why the order was flagged.", "confidence": "low"})
    next_check = "; ".join(checks).capitalize() + "."
    if fields["SoldToParty"] == "CUST-B" and len(parts) == 3:
        next_check += " CUST-B also has order 9100002 blocked."  # the flaw: an order it was never given
    return json.dumps({"likely_cause": "The order has " + " and ".join(parts) + ".",
                       "next_check": next_check, "confidence": "medium"})


def offline_model(version: str):
    from inspect_ai.model import ModelOutput, ModelUsage, get_model

    def write(messages, tools, tool_choice, config):
        fields = json.loads(messages[-1].text.split("Order fields:", 1)[1])
        output = ModelOutput.from_content(model="offline-writer", content=offline_note(version, fields))
        output.usage = ModelUsage(input_tokens=0, output_tokens=0, total_tokens=0)
        return output

    return get_model("mockllm/model", custom_outputs=write, memoize=False)


# ---------------------------------------------------------------- scorers: code checks first, a judge if asked


def parse_note(text: str):
    """The note as a dict, or None. Tolerates a ```json fence around the reply."""
    text = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip())
    try:
        note = json.loads(text)
    except json.JSONDecodeError:
        return None
    return note if isinstance(note, dict) else None


def build_scorers(judge: str | None):
    from inspect_ai.model import get_model
    from inspect_ai.scorer import CORRECT, INCORRECT, Score, accuracy, grouped, scorer, stderr

    metrics = [grouped(accuracy(), "slice"), stderr()]

    def verdict(ok: bool, why: str, answer: str = "") -> Score:
        return Score(value=CORRECT if ok else INCORRECT, answer=answer, explanation=why)

    @scorer(metrics=metrics)
    def valid_format():
        async def score(state, target):
            note = parse_note(state.output.completion)
            ok = bool(note) and {"likely_cause", "next_check", "confidence"} <= set(note) \
                and note.get("confidence") in ("low", "medium", "high")
            return verdict(ok, "JSON with the three fields" if ok else "not the agreed JSON shape",
                           state.output.completion[:200])
        return score

    @scorer(metrics=metrics)
    def covers_facts():
        async def score(state, target):
            note, fields = parse_note(state.output.completion), state.metadata["order"]
            text = json.dumps(note).lower() if note else ""
            missing = []
            if fields["DeliveryBlockReason"] and "delivery" not in text:
                missing.append("delivery block")
            if fields["HeaderBillingBlockReason"] and "billing" not in text:
                missing.append("billing block")
            if fields["TotalCreditCheckStatus"] and "credit" not in text:
                missing.append("credit status")
            no_blocks = not (fields["DeliveryBlockReason"] or fields["HeaderBillingBlockReason"]
                             or fields["TotalCreditCheckStatus"])
            if no_blocks and not re.search(r"does not show|doesn't show|not enough|insufficient", text):
                missing.append("saying the data does not show why")
            return verdict(not missing, "covers every block that is set" if not missing
                           else "misses: " + ", ".join(missing))
        return score

    @scorer(metrics=metrics)
    def no_code_guessing():
        async def score(state, target):
            note = parse_note(state.output.completion) or {}
            cause = str(note.get("likely_cause", "")).lower()
            guess = re.search(r"\b(means|indicates|stands for)\b", cause)
            return verdict(not guess, "explains no code" if not guess else f"guesses a code meaning: {cause}")
        return score

    @scorer(metrics=metrics)
    def no_outside_orders():
        async def score(state, target):
            own = state.metadata["order"]["SalesOrder"]
            others = sorted(set(re.findall(r"\b\d{7,10}\b", state.output.completion)) - {own})
            return verdict(not others, "cites only its own order" if not others
                           else "cites orders it was never given: " + ", ".join(others))
        return score

    chosen = [valid_format(), covers_facts(), no_code_guessing(), no_outside_orders()]
    if judge:
        rubric = ("You grade a triage note about a blocked sales order. A good note uses only the fields given, "
                  "guesses no code meanings, mentions every block that is set and suggests a sensible check. "
                  "Reply with one line: VERDICT: right, VERDICT: partly or VERDICT: wrong.")

        @scorer(metrics=metrics)
        def judge_says_right():
            async def score(state, target):
                reply = await get_model(judge).generate(
                    f"{rubric}\n\nOrder fields:\n{json.dumps(state.metadata['order'], indent=2)}"
                    f"\n\nNote:\n{state.output.completion}")
                found = re.search(r"VERDICT:\s*(right|partly|wrong)", reply.completion, re.I)
                label = found.group(1).lower() if found else "no verdict"
                return verdict(label == "right", f"judge said {label}", reply.completion[:200])
            return score

        chosen.append(judge_says_right())
    return chosen


# ---------------------------------------------------------------- commands


def cmd_init(args) -> None:
    files = {SUITE: "".join(json.dumps(c) + "\n" for c in SAMPLE_SUITE),
             PROMPTS / "v1.txt": PROMPT_V1 + "\n", PROMPTS / "v2.txt": PROMPT_V2 + "\n",
             CONFIG: json.dumps(SAMPLE_CONFIG, indent=2) + "\n"}
    for path, text in files.items():
        if path.exists() and not args.force:
            print(f"Kept     {path.relative_to(HERE.parent).as_posix()} (exists; add --force to overwrite)")
            continue
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(text, encoding="utf-8")
        print(f"Wrote    {path.relative_to(HERE.parent).as_posix()}")
    print(f"\nSuite: {len(read_suite())} cases. Next: python unit08/harness.py run")


def cmd_run(args) -> None:
    load_dotenv()
    for name in (args.model, args.judge):
        if name and name.startswith("anthropic/") and not os.getenv("ANTHROPIC_API_KEY"):
            sys.exit("ANTHROPIC_API_KEY is not set in .env (see Set up your computer). Or leave out --model/--judge.")
    try:
        from inspect_ai import Epochs, Task, eval
        from inspect_ai.dataset import Sample
        from inspect_ai.solver import generate, system_message
    except ImportError:
        sys.exit("inspect-ai is not installed. Run: pip install -r requirements.txt")

    config, cases = read_config(), read_suite()
    version = args.prompt or config["prompt"]
    prompt = read_prompt(version)
    samples = [Sample(id=c["id"], input="Order fields:\n" + json.dumps(c["order"], indent=2), target="",
                      metadata={"slice": c["slice"], "must_pass": c["must_pass"], "order": c["order"]})
               for c in cases]
    task = Task(dataset=samples, solver=[system_message(prompt), generate()],
                scorer=build_scorers(args.judge), name="triage_harness",
                epochs=Epochs(args.epochs, f"at_least_{args.epochs}"))  # a case passes only if it passes every time
    model = args.model or offline_model(version)
    print(f"Running {len(samples)} cases x {args.epochs} epoch(s): prompt {version}, "
          f"writer {args.model or 'offline (made-up)'}{', judge ' + args.judge if args.judge else ''}...")
    log = eval(task, model=model, log_dir=str(LOGS), display="none")[0]
    if log.status != "success":
        sys.exit(f"The run did not finish ({log.status}): {log.error.message if log.error else ''}\n"
                 "Open the log with: inspect view --log-dir unit08/logs/harness")

    per_case = {}  # case id -> {scorer: 1.0 pass or 0.0 fail}, after combining the epochs
    for reduction in log.reductions:
        for s in reduction.samples:
            per_case.setdefault(str(s.sample_id), {})[reduction.scorer] = float(s.value)
    metrics, slices = {}, {}
    for result in log.results.scores:
        values = {k: round(v.value, 3) for k, v in result.metrics.items() if k != "stderr"}
        metrics[result.name] = values.pop("all", None)
        slices[result.name] = values
    record = {
        "run_id": datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S"),
        "prompt": version, "prompt_sha": sha(prompt),
        "writer": args.model or "offline", "judge": args.judge or "", "epochs": args.epochs,
        "suite_sha": sha(SUITE.read_text(encoding="utf-8")), "cases_n": len(cases),
        "git": git_commit(), "metrics": metrics, "slices": slices, "cases": per_case,
        "must_pass": [c["id"] for c in cases if c["must_pass"]],
        "log": Path(log.location).name,
    }
    RESULTS.mkdir(parents=True, exist_ok=True)
    with RUNS.open("a", encoding="utf-8") as handle:
        handle.write(json.dumps(record) + "\n")

    print(f"\nRun {record['run_id']}  prompt {version} ({record['prompt_sha']})  suite {record['suite_sha']}  git {record['git']}")
    print(f"{'scorer':18} {'all':>5}  " + "  ".join(f"{s:>8}" for s in sorted(next(iter(slices.values())))))
    for name, value in metrics.items():
        print(f"{name:18} {value:5.2f}  " + "  ".join(f"{slices[name][s]:8.2f}" for s in sorted(slices[name])))
    failed = [f"{cid} ({', '.join(n for n, v in sc.items() if v < 1)})"
              for cid, sc in per_case.items() if any(v < 1 for v in sc.values())]
    print("\nFailing cases: " + ("; ".join(failed) if failed else "none"))
    print("Saved to unit08/results/runs.jsonl. Next: python unit08/harness.py compare")


def read_runs() -> list:
    if not RUNS.exists():
        sys.exit("No runs recorded yet. Run: python unit08/harness.py run")
    return [json.loads(line) for line in RUNS.read_text(encoding="utf-8").splitlines() if line.strip()]


def cmd_history(args) -> None:
    runs = read_runs()
    names = list(dict.fromkeys(name for r in runs for name in r["metrics"]))  # every scorer seen, in order
    print(f"{'run':16} {'prompt':7} {'writer':10} {'suite':11} {'git':16} " + " ".join(f"{n[:12]:>12}" for n in names))
    for r in runs:
        cells = [f"{r['metrics'][n]:12.2f}" if n in r["metrics"] else f"{'-':>12}" for n in names]
        print(f"{r['run_id']:16} {r['prompt']:7} {r['writer'][:10]:10} {r['suite_sha']:11} {r['git'][:16]:16} "
              + " ".join(cells))


def cmd_baseline(args) -> None:
    runs = read_runs()
    chosen = next((r for r in runs if r["run_id"] == args.run), None) if args.run else runs[-1]
    if not chosen:
        sys.exit(f"No run with id {args.run}. See: python unit08/harness.py history")
    BASELINE.write_text(json.dumps(chosen, indent=2) + "\n", encoding="utf-8")
    print(f"Baseline is now run {chosen['run_id']} (prompt {chosen['prompt']}). Commit unit08/results/baseline.json.")


def gate(run: dict, base: dict, rules: dict) -> list:
    """Return the reasons the run fails the gate; an empty list means it passes."""
    problems = []
    if run["suite_sha"] != base["suite_sha"]:
        problems.append("the suite changed since the baseline, so the numbers aren't comparable: "
                        "run the baseline prompt on the new suite and save it as the baseline first")
    for name, minimum in rules["min"].items():
        value = run["metrics"].get(name)
        if value is None:
            problems.append(f"{name}: not measured in this run")
        elif value < minimum:
            problems.append(f"{name}: {value:.2f} is below the minimum {minimum:.2f}")
    for name, before in base["metrics"].items():
        after = run["metrics"].get(name)
        if after is not None and before - after > rules["max_drop"]:
            problems.append(f"{name}: dropped from {before:.2f} to {after:.2f} (allowed drop {rules['max_drop']:.2f})")
    if rules.get("must_pass_cases"):
        for cid in run["must_pass"]:
            bad = [n for n, v in run["cases"].get(cid, {}).items() if v < 1]
            if bad:
                problems.append(f"must-pass case {cid} failed: {', '.join(bad)}")
    return problems


def cmd_compare(args) -> None:
    if not BASELINE.exists():
        sys.exit("No baseline yet. Run the current prompt, then: python unit08/harness.py baseline")
    run, base = read_runs()[-1], json.loads(BASELINE.read_text(encoding="utf-8"))
    rules = read_config()["gate"]
    lines = [f"# Evaluation report: run {run['run_id']} vs. baseline {base['run_id']}", "",
             f"Prompt {run['prompt']} ({run['prompt_sha']}) vs. {base['prompt']} ({base['prompt_sha']}); "
             f"writer {run['writer']}; suite {run['suite_sha']}; git {run['git']}.", "",
             "| Scorer | Baseline | This run | Change |", "| --- | --- | --- | --- |"]
    for name in run["metrics"]:
        before, after = base["metrics"].get(name), run["metrics"][name]
        change = f"{after - before:+.2f}" if before is not None else "new"
        shown = f"{before:.2f}" if before is not None else "-"
        lines.append(f"| {name} | {shown} | {after:.2f} | {change} |")
    newly, fixed = [], []
    for cid, scores in run["cases"].items():
        for name, value in scores.items():
            was = base["cases"].get(cid, {}).get(name)
            if was == 1 and value < 1:
                newly.append(f"{cid} {name}")
            elif was is not None and was < 1 and value == 1:
                fixed.append(f"{cid} {name}")
    lines += ["", "Newly failing: " + (", ".join(newly) or "none"), "",
              "Newly passing: " + (", ".join(fixed) or "none")]
    problems = gate(run, base, rules)
    lines += ["", "## Gate: " + ("FAIL" if problems else "PASS"), ""] + [f"- {p}" for p in problems]
    text = "\n".join(lines) + "\n"
    REPORT.write_text(text, encoding="utf-8")
    print(re.sub(r"(?m)^#+ ", "", text))
    print("Report saved to unit08/results/report.md")
    sys.exit(1 if problems else 0)


def main() -> None:
    parser = argparse.ArgumentParser(description="Evaluation harness for the triage writer.")
    sub = parser.add_subparsers(dest="command", required=True)
    p = sub.add_parser("init", help="write the suite, prompts and config")
    p.add_argument("--force", action="store_true", help="overwrite existing files")
    p = sub.add_parser("run", help="run the suite and record the results")
    p.add_argument("--prompt", help="prompt version in unit08/prompts, e.g. v2 (default: eval_config.json)")
    p.add_argument("--model", help="a real model writes the notes, e.g. anthropic/claude-opus-5-5")
    p.add_argument("--judge", help="add a model judge as an extra scorer, e.g. anthropic/claude-opus-5-5")
    p.add_argument("--epochs", type=int, default=1, help="run each case this many times; it must pass every time")
    sub.add_parser("history", help="list recorded runs")
    p = sub.add_parser("baseline", help="save a run as the baseline")
    p.add_argument("--run", help="run id (default: the latest)")
    sub.add_parser("compare", help="compare the latest run with the baseline and apply the gate")
    args = parser.parse_args()
    {"init": cmd_init, "run": cmd_run, "history": cmd_history,
     "baseline": cmd_baseline, "compare": cmd_compare}[args.command](args)


if __name__ == "__main__":
    main()

Wrote    unit08/data/triage_suite.jsonl
Wrote    unit08/prompts/v1.txt
Wrote    unit08/prompts/v2.txt
Wrote    unit08/eval_config.json

Suite: 12 cases. Next: python unit08/harness.py run

Running 12 cases x 1 epoch(s): prompt v1, writer offline (made-up)...

Run 20261005-063844  prompt v1 (d6b83c2afe)  suite 00bd24a695  git eea1c41
scorer               all   billing    credit  delivery     multi  no-block
valid_format        1.00      1.00      1.00      1.00      1.00      1.00
covers_facts        1.00      1.00      1.00      1.00      1.00      1.00
no_code_guessing    1.00      1.00      1.00      1.00      1.00      1.00
no_outside_orders   0.92      1.00      1.00      1.00      0.50      1.00

Failing cases: s10 (no_outside_orders)
Saved to unit08/results/runs.jsonl. Next: python unit08/harness.py compare

| Scorer | Baseline | This run | Change |
| --- | --- | --- | --- |
| valid_format | 1.00 | 1.00 | +0.00 |
| covers_facts | 1.00 | 1.00 | +0.00 |
| no_code_guessing | 1.00 | 1.00 | +0.00 |
| no_outside_orders | 0.92 | 0.92 | +0.00 |

Newly failing: none

Newly passing: none

Gate: PASS

Report saved to unit08/results/report.md

Running 12 cases x 1 epoch(s): prompt v2, writer offline (made-up)...

Run 20261005-063848  prompt v2 (f0e7076113)  suite 00bd24a695  git d0657c0
scorer               all   billing    credit  delivery     multi  no-block
valid_format        1.00      1.00      1.00      1.00      1.00      1.00
covers_facts        0.67      1.00      0.50      1.00      0.00      1.00
no_code_guessing    0.83      1.00      0.50      1.00      1.00      1.00
no_outside_orders   1.00      1.00      1.00      1.00      1.00      1.00

Failing cases: s01 (no_code_guessing); s02 (covers_facts); s03 (covers_facts); s04 (no_code_guessing); s09 (covers_facts); s10 (covers_facts)
Saved to unit08/results/runs.jsonl. Next: python unit08/harness.py compare

Newly failing: s01 no_code_guessing, s02 covers_facts, s03 covers_facts, s04 no_code_guessing, s09 covers_facts, s10 covers_facts

Newly passing: s10 no_outside_orders

Gate: FAIL

- covers_facts: 0.67 is below the minimum 0.80
- no_code_guessing: 0.83 is below the minimum 0.90
- covers_facts: dropped from 1.00 to 0.67 (allowed drop 0.05)
- no_code_guessing: dropped from 1.00 to 0.83 (allowed drop 0.05)
- must-pass case s01 failed: no_code_guessing
- must-pass case s04 failed: no_code_guessing

Report saved to unit08/results/report.md
1

# Runs the Unit 8 evaluation harness on GitHub's computers and blocks regressions.
# Uses the offline writer, so it needs no key and costs nothing.
name: unit08-eval

on:
  push:
    paths: ["unit08/**", ".github/workflows/unit08-eval.yml"]
  pull_request:
    paths: ["unit08/**"]
  workflow_dispatch:       # adds a "Run workflow" button on the Actions tab

jobs:
  eval-gate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install the harness libraries
        run: |
          python -m pip install --upgrade pip
          pip install -r unit08/requirements-eval.txt
      - name: Run the suite with the current prompt
        run: python unit08/harness.py run
      - name: Compare with the baseline and apply the gate
        run: python unit08/harness.py compare
      - name: Keep the report
        if: ${{ always() }}
        uses: actions/upload-artifact@v4
        with:
          name: eval-report
          path: unit08/results/report.md