{
  "title": "Agent Safety Eval Lab V2",
  "kind": "classification",
  "dataset_path": "data/processed/classification_examples.jsonl",
  "profile": "full",
  "generated_at": "2026-05-02 07:57:20",
  "git_commit": "1dbdef5",
  "device": {
    "requested_device": "cuda",
    "actual_device": "cuda",
    "cuda_available": true,
    "gpu_name": "NVIDIA GeForce RTX 5090 Laptop GPU",
    "torch_version": "2.10.0+cu128",
    "cuda_runtime": "12.8"
  },
  "results": [
    {
      "accuracy": 0.48888,
      "macro_f1": 0.4480700295242882,
      "safe_precision": 0.4604880883207437,
      "safe_recall": 0.8532208864166517,
      "safe_f1": 0.5981508270960437,
      "safe_support": 5573,
      "unsafe_precision": 0.6237350505979761,
      "unsafe_recall": 0.1957557384148982,
      "unsafe_f1": 0.2979892319525327,
      "unsafe_support": 6927,
      "auroc": 0.524488312415775,
      "experiment_id": "rule_safety_keywords",
      "method": "rule",
      "runtime_seconds": 0.218,
      "rows": 50000
    },
    {
      "accuracy": 0.7752,
      "macro_f1": 0.7743212611538304,
      "safe_precision": 0.7247437774524158,
      "safe_recall": 0.7993899156648124,
      "safe_f1": 0.7602389078498294,
      "safe_support": 5573,
      "unsafe_precision": 0.824020147961593,
      "unsafe_recall": 0.7557384148982244,
      "unsafe_f1": 0.7884036144578314,
      "unsafe_support": 6927,
      "auroc": 0.8592774340368556,
      "experiment_id": "tfidf_word_lr_prompt_response",
      "method": "tfidf_word",
      "runtime_seconds": 4.269,
      "rows": 50000
    },
    {
      "accuracy": 0.75888,
      "macro_f1": 0.7579615706575544,
      "safe_precision": 0.7078122462238103,
      "safe_recall": 0.7819845684550512,
      "safe_f1": 0.7430520034100597,
      "safe_support": 5573,
      "unsafe_precision": 0.8084502601292763,
      "unsafe_recall": 0.7402916125306771,
      "unsafe_f1": 0.772871137905049,
      "unsafe_support": 6927,
      "auroc": 0.8406049698619353,
      "experiment_id": "tfidf_char_lr_prompt_response",
      "method": "tfidf_char",
      "runtime_seconds": 14.322,
      "rows": 50000
    },
    {
      "accuracy": 0.68608,
      "macro_f1": 0.6575838878985079,
      "safe_precision": 0.7482685937970491,
      "safe_recall": 0.4458998743944016,
      "safe_f1": 0.5588036878794693,
      "safe_support": 5573,
      "unsafe_precision": 0.6635799106656498,
      "unsafe_recall": 0.8793128338386026,
      "unsafe_f1": 0.7563640879175463,
      "unsafe_support": 6927,
      "auroc": 0.7993745287264425,
      "experiment_id": "gpu_tfidf_mlp_prompt_response",
      "method": "torch_mlp",
      "runtime_seconds": 5.394,
      "rows": 50000
    },
    {
      "experiment_id": "top_tier_review_gate",
      "artifact_count": 6,
      "coverage_score": 1.0,
      "review_value": "evidence-backed claims, artifact provenance, reproducibility manifest, and strict audit gate"
    }
  ],
  "artifacts": [
    "reports\\results\\v2_main_results.csv",
    "reports\\results\\v2_ablation_results.csv",
    "reports\\results\\v2_failure_cases.json",
    "reports\\figures\\v2_accuracy_by_experiment.png",
    "reports\\figures\\v2_confusion_matrix.png",
    "reports\\figures\\v2_model_macro_f1.png",
    "docs/top_tier_reviewer_packet.md",
    "reports/results/claim_evidence_matrix.csv",
    "reports/results/artifact_manifest.json",
    "reports/results/reproducibility_manifest.json",
    "reports/results/top_tier_quality_gate.json",
    "reports/figures/top_tier_review_scores.png"
  ],
  "discussion": "Agent safety needs both local prompt/response classification and trace-level policy interpretation. V2 measures where classifiers miss unsafe responses and how those misses propagate into simulated tool decisions.",
  "total_runtime_seconds": 24.71
}