From 02c0586b43bccd8d00f98e11d362380ec9a34148 Mon Sep 17 00:00:00 2001 From: Daniel Maddern Date: Mon, 17 Aug 2026 13:22:52 +0700 Subject: [PATCH] Use strict type and source file scoring --- .../forgeguard_evmbench_source_path.patch | 15 ++ scripts/run_guard3b_overnight_spark.sh | 18 +- scripts/score_evmbench_type_source_file.py | 164 ++++++++++++++++++ 3 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 scripts/patches/forgeguard_evmbench_source_path.patch create mode 100644 scripts/score_evmbench_type_source_file.py diff --git a/scripts/patches/forgeguard_evmbench_source_path.patch b/scripts/patches/forgeguard_evmbench_source_path.patch new file mode 100644 index 0000000..ffbb1cc --- /dev/null +++ b/scripts/patches/forgeguard_evmbench_source_path.patch @@ -0,0 +1,15 @@ +diff --git a/scripts/run_evmbench.py b/scripts/run_evmbench.py +--- a/scripts/run_evmbench.py ++++ b/scripts/run_evmbench.py +@@ -598,7 +598,13 @@ def run_benchmark(model, tokenizer, configs_dir, contracts_dir, use_processor=F + ) + all_raw.append(raw) +- all_findings.extend(findings) ++ # Findings are produced per source file, but the legacy result ++ # aggregated them without retaining that association. ++ for finding in findings: ++ finding_with_source = dict(finding) ++ finding_with_source["source_path"] = sol_path ++ all_findings.append(finding_with_source) + if findings: + detected = True diff --git a/scripts/run_guard3b_overnight_spark.sh b/scripts/run_guard3b_overnight_spark.sh index f72b895..d36ebf5 100755 --- a/scripts/run_guard3b_overnight_spark.sh +++ b/scripts/run_guard3b_overnight_spark.sh @@ -4,6 +4,7 @@ set -euo pipefail ROOT=/home/daniel/forgeguard PROJECT="$ROOT/project" +ARTIFEX_ROOT=/home/daniel/Artifex DATA="$ROOT/artifex_dataset_versions/guard_curated_v03/training.json" RUN_ROOT="$PROJECT/artifex_overnight/guard3b_v03_overnight" CANARY=artifex-guard3b-v03-canary @@ -16,24 +17,34 @@ mkdir -p "$RUN_ROOT" docker_run() { docker run --rm --gpus all --shm-size=16g \ -v "$PROJECT:/workspace" \ + -v "$ARTIFEX_ROOT/scripts/score_evmbench_type_source_file.py:/workspace/artifex/score_evmbench_type_source_file.py:ro" \ -v "$ROOT/artifex_dataset_versions:/workspace/artifex_dataset_versions" \ -v /home/daniel/.cache/huggingface:/root/.cache/huggingface \ "$IMAGE" "$@" } +install_evaluator_contract() { + if ! grep -q 'finding_with_source\["source_path"\]' "$PROJECT/scripts/run_evmbench.py"; then + patch --batch --forward -d "$PROJECT" -p1 < "$ARTIFEX_ROOT/scripts/patches/forgeguard_evmbench_source_path.patch" + fi +} + evaluate() { local name=$1 adapter=${2:-} local result="$RUN_ROOT/${name}_evmbench.json" score="$RUN_ROOT/${name}_strict.json" local args=(python /workspace/scripts/run_evmbench.py --model "$MODEL" --unsloth --save "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json") if [[ -n "$adapter" ]]; then args+=(--adapter "/workspace/artifex_overnight/$adapter"); fi docker_run "${args[@]}" > "$RUN_ROOT/${name}_evaluation.log" 2>&1 - docker_run python /workspace/scripts/score_evmbench_strict.py "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" > "$score" 2>&1 + docker_run python /workspace/artifex/score_evmbench_type_source_file.py \ + "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_evmbench.json" \ + --output "/workspace/artifex_overnight/guard3b_v03_overnight/${name}_strict.json" \ + > "$RUN_ROOT/${name}_scoring.log" 2>&1 [[ -s "$result" && -s "$score" ]] } qwen_choice() { local summary - summary=$(tail -c 10000 "$RUN_ROOT/base_strict.json" 2>/dev/null; tail -c 10000 "$RUN_ROOT/current_strict.json" 2>/dev/null || true) + summary=$(jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/base_strict.json"; jq -c '{acceptance_metric,summary:{targets,canonical_type_and_target_source_file_match}}' "$RUN_ROOT/current_strict.json") local body body=$(jq -n --arg prompt "You are MODEL_DIRECTOR for a bounded Guard 3B overnight run. Same-harness strict-score evidence follows: $summary Choose one JSON decision only: {\"action\":\"CONTINUE\"|\"REPLICATE\"|\"STOP\",\"rationale\":\"...\"}. CONTINUE means resume the current adapter with lower LR; REPLICATE means fresh seed from the immutable base; STOP means evidence/remaining time does not justify another run." '{model:"qwen38",messages:[{role:"user",content:$prompt}],temperature:0,max_tokens:200}') curl -fsS --max-time 180 http://127.0.0.1:8002/v1/chat/completions -H 'Content-Type: application/json' -d "$body" | jq -r '.choices[0].message.content' > "$RUN_ROOT/qwen_plan.json" || true @@ -55,10 +66,11 @@ if [[ "$(cat "$RUN_ROOT/canary_exit_code.txt")" != "0" ]]; then exit 1 fi cp "$PROJECT/artifex_overnight/guard3b_v03_canary/pilot_metrics.json" "$RUN_ROOT/canary_metrics.json" +install_evaluator_contract # Base and the initial Challenger use the identical fixed harness/version. # Reuse a completed base result when resuming after an infrastructure repair. -if [[ ! -s "$RUN_ROOT/base_evmbench.json" ]]; then evaluate base; else docker_run python /workspace/scripts/score_evmbench_strict.py "/workspace/artifex_overnight/guard3b_v03_overnight/base_evmbench.json" > "$RUN_ROOT/base_strict.json" 2>&1; fi +if [[ ! -s "$RUN_ROOT/base_evmbench.json" ]]; then evaluate base; else docker_run python /workspace/artifex/score_evmbench_type_source_file.py "/workspace/artifex_overnight/guard3b_v03_overnight/base_evmbench.json" --output "/workspace/artifex_overnight/guard3b_v03_overnight/base_strict.json" > "$RUN_ROOT/base_scoring.log" 2>&1; fi cp "$RUN_ROOT/base_strict.json" "$RUN_ROOT/current_strict.json" CURRENT=guard3b_v03_canary evaluate canary "$CURRENT" diff --git a/scripts/score_evmbench_type_source_file.py b/scripts/score_evmbench_type_source_file.py new file mode 100644 index 0000000..40ac310 --- /dev/null +++ b/scripts/score_evmbench_type_source_file.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Score EVMBench by canonical vulnerability type and patch-mapped source file. + +This is the acceptance scorer for the Guard 3B program. It deliberately does +not score the legacy "any finding" detection rate. EVMBench's runnable issue +inventory has a reviewed patch-mapped source file but no reviewed function or +line anchor, so function/line accuracy is reported as unavailable rather than +being guessed from model text. +""" +import argparse +import json +import re +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_INVENTORY = ROOT / "docs" / "evmbench_runnable_issue_inventory.json" + +TYPE_ALIASES = { + "reentrancy": "reentrancy", + "re-entrancy": "reentrancy", + "flash loan": "flash_loan", + "flash_loan": "flash_loan", + "oracle manipulation": "oracle_manipulation", + "oracle_manipulation": "oracle_manipulation", + "access control": "access_control", + "access_control": "access_control", + "integer overflow": "integer_overflow", + "integer_overflow": "integer_overflow", + "integer underflow": "integer_overflow", + "front running": "front_running", + "front-running": "front_running", + "front_running": "front_running", + "delegate call": "delegate_call", + "delegatecall": "delegate_call", + "delegate_call": "delegate_call", + "logic error": "logic_error", + "logic_error": "logic_error", +} + + +def read_json(path): + with path.open(encoding="utf-8") as handle: + return json.load(handle) + + +def normalize_type(value): + value = re.sub(r"[ _-]+", " ", str(value or "").strip().lower()) + return TYPE_ALIASES.get(value, "unknown") + + +def normalize_path(value): + return str(value or "").replace("\\", "/").strip("/").lower() + + +def source_file_matches(reported_path, expected_paths): + reported = normalize_path(reported_path) + return bool(reported) and any( + reported.endswith(normalize_path(expected)) for expected in expected_paths + ) + + +def rate(count, total): + return round(100 * count / total, 1) if total else 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Score canonical type plus patch-mapped target source file" + ) + parser.add_argument("results", type=Path) + parser.add_argument("--inventory", type=Path, default=DEFAULT_INVENTORY) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + result = read_json(args.results) + inventory = read_json(args.inventory) + records = result.get("vulnerabilities") + if not isinstance(records, list): + raise ValueError("results lacks per-vulnerability findings") + by_key = { + f"{record.get('audit_id')}:{record.get('vuln_id')}": record + for record in records + } + rows = [] + missing_source_path = 0 + for issue in inventory.get("issues", []): + key = f"{issue['audit_id']}:{issue['vulnerability_id']}" + record = by_key.get(key, {}) + expected_paths = [ + source["path"] for source in issue.get("source_files", []) + if source.get("exists") + ] + findings = record.get("findings", []) + if not isinstance(findings, list): + findings = [] + annotated = [] + for finding in findings: + source_path = finding.get("source_path", "") + if not source_path: + missing_source_path += 1 + type_match = normalize_type(finding.get("type")) == issue["canonical_type"] + file_match = source_file_matches(source_path, expected_paths) + annotated.append({ + "reported_type": finding.get("type", ""), + "normalized_type": normalize_type(finding.get("type")), + "reported_source_path": source_path, + "reported_location": finding.get("location", ""), + "canonical_type_match": type_match, + "target_source_file_match": file_match, + "canonical_type_and_target_source_file_match": type_match and file_match, + }) + rows.append({ + "target": key, + "canonical_type": issue["canonical_type"], + "expected_source_files": expected_paths, + "any_finding": bool(findings), + "canonical_type_match": any(item["canonical_type_match"] for item in annotated), + "canonical_type_and_target_source_file_match": any( + item["canonical_type_and_target_source_file_match"] + for item in annotated + ), + "findings": annotated, + }) + + total = len(rows) + canonical_count = sum(row["canonical_type_match"] for row in rows) + strict_count = sum( + row["canonical_type_and_target_source_file_match"] for row in rows + ) + artifact = { + "schema_version": 1, + "scorer": "score_evmbench_type_source_file.py", + "input_result": str(args.results), + "inventory": str(args.inventory), + "acceptance_metric": "canonical_type_and_target_source_file_match", + "metric_definition": ( + "A finding must match the inventory canonical type and be produced " + "while evaluating an inventory patch-mapped target source file." + ), + "location_label_coverage": "target source file only; reviewed function/line anchors are unavailable", + "summary": { + "targets": total, + "canonical_type_match": {"count": canonical_count, "rate_percent": rate(canonical_count, total)}, + "canonical_type_and_target_source_file_match": { + "count": strict_count, + "rate_percent": rate(strict_count, total), + }, + }, + "validation": { + "findings_missing_source_path": missing_source_path, + "legacy_any_finding_excluded_from_acceptance": True, + }, + "rows": rows, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", encoding="utf-8") as handle: + json.dump(artifact, handle, indent=2) + handle.write("\n") + print(json.dumps(artifact["summary"], sort_keys=True)) + + +if __name__ == "__main__": + main()