{
  "schema": "moderation-analysis-v1",
  "tasks": [
    {
      "slug": "aegis_prompt",
      "n": 1928,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_prompt_base.json",
          "input_sha256": "e40c88c472b7423130bc22431de0ac2ede2686ad5ebfade059bc9f5a9ab8e42b",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1928,
            "correct": 1183,
            "accuracy": 0.6135892116182573,
            "wilson_95": [
              0.591648865344231,
              0.6350778145290646
            ],
            "macro_f1": 0.6134310350535019,
            "errors": 0,
            "probability_rows": 1928,
            "multiclass_brier": 0.46984878897374927,
            "nll": 0.6685138332136695,
            "ece_10_equal_width": 0.07882407101094867,
            "p50_ms": 28.695458080619574,
            "p95_ms": 158.2837919704616,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1928,
            "per_class": {
              "safe": {
                "support": 889,
                "negative_support": 1039,
                "tp": 611,
                "fp": 467,
                "fn": 278,
                "tn": 572,
                "precision": 0.5667903525046383,
                "recall": 0.6872890888638921,
                "recall_wilson_95": [
                  0.6560635995520714,
                  0.7169029520234365
                ],
                "false_positive_rate": 0.4494706448508181,
                "false_positive_wilson_95": [
                  0.41946503958655623,
                  0.4798485146341016
                ]
              },
              "unsafe": {
                "support": 1039,
                "negative_support": 889,
                "tp": 572,
                "fp": 278,
                "fn": 467,
                "tn": 611,
                "precision": 0.6729411764705883,
                "recall": 0.5505293551491819,
                "recall_wilson_95": [
                  0.5201514853658984,
                  0.5805349604134438
                ],
                "false_positive_rate": 0.312710911136108,
                "false_positive_wilson_95": [
                  0.28309704797656354,
                  0.3439364004479287
                ]
              }
            },
            "majority_baseline": 0.5389004149377593,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 572,
              "fp": 278,
              "fn": 467,
              "tn": 611,
              "precision": 0.6729411764705883,
              "recall": 0.5505293551491819,
              "recall_wilson_95": [
                0.5201514853658984,
                0.5805349604134438
              ],
              "false_positive_rate": 0.312710911136108,
              "false_positive_wilson_95": [
                0.28309704797656354,
                0.3439364004479287
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 572,
              "fp": 278,
              "fn": 467,
              "tn": 611,
              "precision": 0.6729411764705883,
              "recall": 0.5505293551491819,
              "recall_wilson_95": [
                0.5201514853658984,
                0.5805349604134438
              ],
              "false_positive_rate": 0.312710911136108,
              "false_positive_wilson_95": [
                0.28309704797656354,
                0.3439364004479287
              ],
              "f1": 0.6056114346214928,
              "negative_errors": 0
            },
            "harmful_f1": 0.6056114346214928,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.6918532608951758,
              "scored_rows": 1928,
              "excluded_rows": 0,
              "positive_support": 1039,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "Controlled/Regulated Substances": {
                "support": 112,
                "tp": 64,
                "recall": 0.5714285714285714,
                "recall_wilson_95": [
                  0.478911377409721,
                  0.659208430152144
                ]
              },
              "Copyright/Trademark/Plagiarism": {
                "support": 4,
                "tp": 2,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.15003898915214947,
                  0.8499610108478506
                ]
              },
              "Criminal Planning/Confessions": {
                "support": 502,
                "tp": 329,
                "recall": 0.6553784860557769,
                "recall_wilson_95": [
                  0.6127666578121234,
                  0.6956303651675557
                ]
              },
              "Fraud/Deception": {
                "support": 14,
                "tp": 6,
                "recall": 0.42857142857142855,
                "recall_wilson_95": [
                  0.21380798904474121,
                  0.6740935542034926
                ]
              },
              "Guns and Illegal Weapons": {
                "support": 62,
                "tp": 40,
                "recall": 0.6451612903225806,
                "recall_wilson_95": [
                  0.5208113075778423,
                  0.752572671489501
                ]
              },
              "Harassment": {
                "support": 152,
                "tp": 80,
                "recall": 0.5263157894736842,
                "recall_wilson_95": [
                  0.44727197623771375,
                  0.6040622454805551
                ]
              },
              "Hate/Identity Hate": {
                "support": 175,
                "tp": 76,
                "recall": 0.4342857142857143,
                "recall_wilson_95": [
                  0.36303942685498813,
                  0.5083550464865666
                ]
              },
              "High Risk Gov Decision Making": {
                "support": 3,
                "tp": 3,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.4385029682449546,
                  1.0
                ]
              },
              "Illegal Activity": {
                "support": 12,
                "tp": 12,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.7575059933447592,
                  1.0
                ]
              },
              "Immoral/Unethical": {
                "support": 14,
                "tp": 12,
                "recall": 0.8571428571428571,
                "recall_wilson_95": [
                  0.6005862050302965,
                  0.9599060787285351
                ]
              },
              "Malware": {
                "support": 8,
                "tp": 5,
                "recall": 0.625,
                "recall_wilson_95": [
                  0.30574239460262737,
                  0.8631557141764026
                ]
              },
              "Manipulation": {
                "support": 2,
                "tp": 1,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.09453120573423074,
                  0.9054687942657693
                ]
              },
              "Needs Caution": {
                "support": 69,
                "tp": 28,
                "recall": 0.4057971014492754,
                "recall_wilson_95": [
                  0.29788908641297196,
                  0.523641121651632
                ]
              },
              "Other": {
                "support": 14,
                "tp": 10,
                "recall": 0.7142857142857143,
                "recall_wilson_95": [
                  0.4535091567010681,
                  0.8827862135542308
                ]
              },
              "PII/Privacy": {
                "support": 99,
                "tp": 33,
                "recall": 0.3333333333333333,
                "recall_wilson_95": [
                  0.24823816410788008,
                  0.4308795732480624
                ]
              },
              "Political/Misinformation/Conspiracy": {
                "support": 21,
                "tp": 13,
                "recall": 0.6190476190476191,
                "recall_wilson_95": [
                  0.4087865377511576,
                  0.7924898863143551
                ]
              },
              "Profanity": {
                "support": 98,
                "tp": 59,
                "recall": 0.6020408163265306,
                "recall_wilson_95": [
                  0.5030495272292229,
                  0.6933341482561485
                ]
              },
              "Sexual": {
                "support": 99,
                "tp": 67,
                "recall": 0.6767676767676768,
                "recall_wilson_95": [
                  0.5795296789352458,
                  0.7607999935660275
                ]
              },
              "Sexual (minor)": {
                "support": 11,
                "tp": 5,
                "recall": 0.45454545454545453,
                "recall_wilson_95": [
                  0.21271271622459764,
                  0.719908462590678
                ]
              },
              "Suicide and Self Harm": {
                "support": 38,
                "tp": 21,
                "recall": 0.5526315789473685,
                "recall_wilson_95": [
                  0.3970629712199248,
                  0.6985359900194876
                ]
              },
              "Threat": {
                "support": 16,
                "tp": 13,
                "recall": 0.8125,
                "recall_wilson_95": [
                  0.5699111903802586,
                  0.9340840092857187
                ]
              },
              "Unauthorized Advice": {
                "support": 23,
                "tp": 11,
                "recall": 0.4782608695652174,
                "recall_wilson_95": [
                  0.29237157416646653,
                  0.6703726266357259
                ]
              },
              "Violence": {
                "support": 176,
                "tp": 125,
                "recall": 0.7102272727272727,
                "recall_wilson_95": [
                  0.6392822662609565,
                  0.7721912642878566
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1928,
            "summary": {
              "n": 1928,
              "correct": 1183,
              "accuracy": 0.6135892116182573,
              "wilson_95": [
                0.591648865344231,
                0.6350778145290646
              ],
              "macro_f1": 0.6134310350535019,
              "errors": 0,
              "probability_rows": 1928,
              "multiclass_brier": 0.46984878897374927,
              "nll": 0.6685138332136695,
              "ece_10_equal_width": 0.07882407101094867,
              "p50_ms": 28.695458080619574,
              "p95_ms": 158.2837919704616,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1928,
              "per_class": {
                "safe": {
                  "support": 889,
                  "negative_support": 1039,
                  "tp": 611,
                  "fp": 467,
                  "fn": 278,
                  "tn": 572,
                  "precision": 0.5667903525046383,
                  "recall": 0.6872890888638921,
                  "recall_wilson_95": [
                    0.6560635995520714,
                    0.7169029520234365
                  ],
                  "false_positive_rate": 0.4494706448508181,
                  "false_positive_wilson_95": [
                    0.41946503958655623,
                    0.4798485146341016
                  ]
                },
                "unsafe": {
                  "support": 1039,
                  "negative_support": 889,
                  "tp": 572,
                  "fp": 278,
                  "fn": 467,
                  "tn": 611,
                  "precision": 0.6729411764705883,
                  "recall": 0.5505293551491819,
                  "recall_wilson_95": [
                    0.5201514853658984,
                    0.5805349604134438
                  ],
                  "false_positive_rate": 0.312710911136108,
                  "false_positive_wilson_95": [
                    0.28309704797656354,
                    0.3439364004479287
                  ]
                }
              },
              "majority_baseline": 0.5389004149377593,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 572,
                "fp": 278,
                "fn": 467,
                "tn": 611,
                "precision": 0.6729411764705883,
                "recall": 0.5505293551491819,
                "recall_wilson_95": [
                  0.5201514853658984,
                  0.5805349604134438
                ],
                "false_positive_rate": 0.312710911136108,
                "false_positive_wilson_95": [
                  0.28309704797656354,
                  0.3439364004479287
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 572,
                "fp": 278,
                "fn": 467,
                "tn": 611,
                "precision": 0.6729411764705883,
                "recall": 0.5505293551491819,
                "recall_wilson_95": [
                  0.5201514853658984,
                  0.5805349604134438
                ],
                "false_positive_rate": 0.312710911136108,
                "false_positive_wilson_95": [
                  0.28309704797656354,
                  0.3439364004479287
                ],
                "f1": 0.6056114346214928,
                "negative_errors": 0
              },
              "harmful_f1": 0.6056114346214928,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.6918532608951758,
                "scored_rows": 1928,
                "excluded_rows": 0,
                "positive_support": 1039,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "Controlled/Regulated Substances": {
                  "support": 112,
                  "tp": 64,
                  "recall": 0.5714285714285714,
                  "recall_wilson_95": [
                    0.478911377409721,
                    0.659208430152144
                  ]
                },
                "Copyright/Trademark/Plagiarism": {
                  "support": 4,
                  "tp": 2,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.15003898915214947,
                    0.8499610108478506
                  ]
                },
                "Criminal Planning/Confessions": {
                  "support": 502,
                  "tp": 329,
                  "recall": 0.6553784860557769,
                  "recall_wilson_95": [
                    0.6127666578121234,
                    0.6956303651675557
                  ]
                },
                "Fraud/Deception": {
                  "support": 14,
                  "tp": 6,
                  "recall": 0.42857142857142855,
                  "recall_wilson_95": [
                    0.21380798904474121,
                    0.6740935542034926
                  ]
                },
                "Guns and Illegal Weapons": {
                  "support": 62,
                  "tp": 40,
                  "recall": 0.6451612903225806,
                  "recall_wilson_95": [
                    0.5208113075778423,
                    0.752572671489501
                  ]
                },
                "Harassment": {
                  "support": 152,
                  "tp": 80,
                  "recall": 0.5263157894736842,
                  "recall_wilson_95": [
                    0.44727197623771375,
                    0.6040622454805551
                  ]
                },
                "Hate/Identity Hate": {
                  "support": 175,
                  "tp": 76,
                  "recall": 0.4342857142857143,
                  "recall_wilson_95": [
                    0.36303942685498813,
                    0.5083550464865666
                  ]
                },
                "High Risk Gov Decision Making": {
                  "support": 3,
                  "tp": 3,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.4385029682449546,
                    1.0
                  ]
                },
                "Illegal Activity": {
                  "support": 12,
                  "tp": 12,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.7575059933447592,
                    1.0
                  ]
                },
                "Immoral/Unethical": {
                  "support": 14,
                  "tp": 12,
                  "recall": 0.8571428571428571,
                  "recall_wilson_95": [
                    0.6005862050302965,
                    0.9599060787285351
                  ]
                },
                "Malware": {
                  "support": 8,
                  "tp": 5,
                  "recall": 0.625,
                  "recall_wilson_95": [
                    0.30574239460262737,
                    0.8631557141764026
                  ]
                },
                "Manipulation": {
                  "support": 2,
                  "tp": 1,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.09453120573423074,
                    0.9054687942657693
                  ]
                },
                "Needs Caution": {
                  "support": 69,
                  "tp": 28,
                  "recall": 0.4057971014492754,
                  "recall_wilson_95": [
                    0.29788908641297196,
                    0.523641121651632
                  ]
                },
                "Other": {
                  "support": 14,
                  "tp": 10,
                  "recall": 0.7142857142857143,
                  "recall_wilson_95": [
                    0.4535091567010681,
                    0.8827862135542308
                  ]
                },
                "PII/Privacy": {
                  "support": 99,
                  "tp": 33,
                  "recall": 0.3333333333333333,
                  "recall_wilson_95": [
                    0.24823816410788008,
                    0.4308795732480624
                  ]
                },
                "Political/Misinformation/Conspiracy": {
                  "support": 21,
                  "tp": 13,
                  "recall": 0.6190476190476191,
                  "recall_wilson_95": [
                    0.4087865377511576,
                    0.7924898863143551
                  ]
                },
                "Profanity": {
                  "support": 98,
                  "tp": 59,
                  "recall": 0.6020408163265306,
                  "recall_wilson_95": [
                    0.5030495272292229,
                    0.6933341482561485
                  ]
                },
                "Sexual": {
                  "support": 99,
                  "tp": 67,
                  "recall": 0.6767676767676768,
                  "recall_wilson_95": [
                    0.5795296789352458,
                    0.7607999935660275
                  ]
                },
                "Sexual (minor)": {
                  "support": 11,
                  "tp": 5,
                  "recall": 0.45454545454545453,
                  "recall_wilson_95": [
                    0.21271271622459764,
                    0.719908462590678
                  ]
                },
                "Suicide and Self Harm": {
                  "support": 38,
                  "tp": 21,
                  "recall": 0.5526315789473685,
                  "recall_wilson_95": [
                    0.3970629712199248,
                    0.6985359900194876
                  ]
                },
                "Threat": {
                  "support": 16,
                  "tp": 13,
                  "recall": 0.8125,
                  "recall_wilson_95": [
                    0.5699111903802586,
                    0.9340840092857187
                  ]
                },
                "Unauthorized Advice": {
                  "support": 23,
                  "tp": 11,
                  "recall": 0.4782608695652174,
                  "recall_wilson_95": [
                    0.29237157416646653,
                    0.6703726266357259
                  ]
                },
                "Violence": {
                  "support": 176,
                  "tp": 125,
                  "recall": 0.7102272727272727,
                  "recall_wilson_95": [
                    0.6392822662609565,
                    0.7721912642878566
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_prompt_fine1000.json",
          "input_sha256": "8f61612fe5fbddd38587f7099b1a8862fef760da477678ee252f2d59b77f4052",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1928,
            "correct": 1527,
            "accuracy": 0.79201244813278,
            "wilson_95": [
              0.7733237849789271,
              0.8095397801747964
            ],
            "macro_f1": 0.7886551480467505,
            "errors": 0,
            "probability_rows": 1928,
            "multiclass_brier": 0.3134374106224066,
            "nll": 0.5040660071501525,
            "ece_10_equal_width": 0.07585238589211622,
            "p50_ms": 28.01916585303843,
            "p95_ms": 68.82058293558657,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1928,
            "per_class": {
              "safe": {
                "support": 889,
                "negative_support": 1039,
                "tp": 642,
                "fp": 154,
                "fn": 247,
                "tn": 885,
                "precision": 0.8065326633165829,
                "recall": 0.7221597300337458,
                "recall_wilson_95": [
                  0.6918067619851456,
                  0.7506010094032194
                ],
                "false_positive_rate": 0.1482194417709336,
                "false_positive_wilson_95": [
                  0.1279110846740561,
                  0.17111946899547548
                ]
              },
              "unsafe": {
                "support": 1039,
                "negative_support": 889,
                "tp": 885,
                "fp": 247,
                "fn": 154,
                "tn": 642,
                "precision": 0.7818021201413428,
                "recall": 0.8517805582290664,
                "recall_wilson_95": [
                  0.8288805310045245,
                  0.872088915325944
                ],
                "false_positive_rate": 0.2778402699662542,
                "false_positive_wilson_95": [
                  0.2493989905967806,
                  0.3081932380148543
                ]
              }
            },
            "majority_baseline": 0.5389004149377593,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 885,
              "fp": 247,
              "fn": 154,
              "tn": 642,
              "precision": 0.7818021201413428,
              "recall": 0.8517805582290664,
              "recall_wilson_95": [
                0.8288805310045245,
                0.872088915325944
              ],
              "false_positive_rate": 0.2778402699662542,
              "false_positive_wilson_95": [
                0.2493989905967806,
                0.3081932380148543
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 885,
              "fp": 247,
              "fn": 154,
              "tn": 642,
              "precision": 0.7818021201413428,
              "recall": 0.8517805582290664,
              "recall_wilson_95": [
                0.8288805310045245,
                0.872088915325944
              ],
              "false_positive_rate": 0.2778402699662542,
              "false_positive_wilson_95": [
                0.2493989905967806,
                0.3081932380148543
              ],
              "f1": 0.8152924919391985,
              "negative_errors": 0
            },
            "harmful_f1": 0.8152924919391985,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8693378278475258,
              "scored_rows": 1928,
              "excluded_rows": 0,
              "positive_support": 1039,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "Controlled/Regulated Substances": {
                "support": 112,
                "tp": 100,
                "recall": 0.8928571428571429,
                "recall_wilson_95": [
                  0.8220190653154164,
                  0.9376398762748422
                ]
              },
              "Copyright/Trademark/Plagiarism": {
                "support": 4,
                "tp": 2,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.15003898915214947,
                  0.8499610108478506
                ]
              },
              "Criminal Planning/Confessions": {
                "support": 502,
                "tp": 478,
                "recall": 0.952191235059761,
                "recall_wilson_95": [
                  0.9298494365746138,
                  0.9676649764559907
                ]
              },
              "Fraud/Deception": {
                "support": 14,
                "tp": 11,
                "recall": 0.7857142857142857,
                "recall_wilson_95": [
                  0.524107694133997,
                  0.9242861328730683
                ]
              },
              "Guns and Illegal Weapons": {
                "support": 62,
                "tp": 57,
                "recall": 0.9193548387096774,
                "recall_wilson_95": [
                  0.8247142865138739,
                  0.9650616530140073
                ]
              },
              "Harassment": {
                "support": 152,
                "tp": 133,
                "recall": 0.875,
                "recall_wilson_95": [
                  0.8130162507557277,
                  0.9184964087296055
                ]
              },
              "Hate/Identity Hate": {
                "support": 175,
                "tp": 155,
                "recall": 0.8857142857142857,
                "recall_wilson_95": [
                  0.8300698699991705,
                  0.9247886560395292
                ]
              },
              "High Risk Gov Decision Making": {
                "support": 3,
                "tp": 3,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.4385029682449546,
                  1.0
                ]
              },
              "Illegal Activity": {
                "support": 12,
                "tp": 12,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.7575059933447592,
                  1.0
                ]
              },
              "Immoral/Unethical": {
                "support": 14,
                "tp": 12,
                "recall": 0.8571428571428571,
                "recall_wilson_95": [
                  0.6005862050302965,
                  0.9599060787285351
                ]
              },
              "Malware": {
                "support": 8,
                "tp": 6,
                "recall": 0.75,
                "recall_wilson_95": [
                  0.40927543031016883,
                  0.9285207872478909
                ]
              },
              "Manipulation": {
                "support": 2,
                "tp": 1,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.09453120573423074,
                  0.9054687942657693
                ]
              },
              "Needs Caution": {
                "support": 69,
                "tp": 53,
                "recall": 0.7681159420289855,
                "recall_wilson_95": [
                  0.6560315295999074,
                  0.8519209551392967
                ]
              },
              "Other": {
                "support": 14,
                "tp": 14,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.7846891972623643,
                  1.0
                ]
              },
              "PII/Privacy": {
                "support": 99,
                "tp": 73,
                "recall": 0.7373737373737373,
                "recall_wilson_95": [
                  0.6429956284754338,
                  0.814018503169133
                ]
              },
              "Political/Misinformation/Conspiracy": {
                "support": 21,
                "tp": 15,
                "recall": 0.7142857142857143,
                "recall_wilson_95": [
                  0.5004361928391441,
                  0.8618613704787788
                ]
              },
              "Profanity": {
                "support": 98,
                "tp": 80,
                "recall": 0.8163265306122449,
                "recall_wilson_95": [
                  0.7282500277598462,
                  0.8805393662448048
                ]
              },
              "Sexual": {
                "support": 99,
                "tp": 89,
                "recall": 0.898989898989899,
                "recall_wilson_95": [
                  0.8239693643888216,
                  0.9442033249711951
                ]
              },
              "Sexual (minor)": {
                "support": 11,
                "tp": 10,
                "recall": 0.9090909090909091,
                "recall_wilson_95": [
                  0.6226415635484043,
                  0.9837678271141151
                ]
              },
              "Suicide and Self Harm": {
                "support": 38,
                "tp": 31,
                "recall": 0.8157894736842105,
                "recall_wilson_95": [
                  0.665811419288686,
                  0.9077823481477872
                ]
              },
              "Threat": {
                "support": 16,
                "tp": 16,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8063923194655637,
                  1.0000000000000002
                ]
              },
              "Unauthorized Advice": {
                "support": 23,
                "tp": 2,
                "recall": 0.08695652173913043,
                "recall_wilson_95": [
                  0.024180004484220363,
                  0.26795981075743663
                ]
              },
              "Violence": {
                "support": 176,
                "tp": 167,
                "recall": 0.9488636363636364,
                "recall_wilson_95": [
                  0.9056847740581433,
                  0.972866818194728
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1928,
            "summary": {
              "n": 1928,
              "correct": 1527,
              "accuracy": 0.79201244813278,
              "wilson_95": [
                0.7733237849789271,
                0.8095397801747964
              ],
              "macro_f1": 0.7886551480467505,
              "errors": 0,
              "probability_rows": 1928,
              "multiclass_brier": 0.3134374106224066,
              "nll": 0.5040660071501525,
              "ece_10_equal_width": 0.07585238589211622,
              "p50_ms": 28.01916585303843,
              "p95_ms": 68.82058293558657,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1928,
              "per_class": {
                "safe": {
                  "support": 889,
                  "negative_support": 1039,
                  "tp": 642,
                  "fp": 154,
                  "fn": 247,
                  "tn": 885,
                  "precision": 0.8065326633165829,
                  "recall": 0.7221597300337458,
                  "recall_wilson_95": [
                    0.6918067619851456,
                    0.7506010094032194
                  ],
                  "false_positive_rate": 0.1482194417709336,
                  "false_positive_wilson_95": [
                    0.1279110846740561,
                    0.17111946899547548
                  ]
                },
                "unsafe": {
                  "support": 1039,
                  "negative_support": 889,
                  "tp": 885,
                  "fp": 247,
                  "fn": 154,
                  "tn": 642,
                  "precision": 0.7818021201413428,
                  "recall": 0.8517805582290664,
                  "recall_wilson_95": [
                    0.8288805310045245,
                    0.872088915325944
                  ],
                  "false_positive_rate": 0.2778402699662542,
                  "false_positive_wilson_95": [
                    0.2493989905967806,
                    0.3081932380148543
                  ]
                }
              },
              "majority_baseline": 0.5389004149377593,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 885,
                "fp": 247,
                "fn": 154,
                "tn": 642,
                "precision": 0.7818021201413428,
                "recall": 0.8517805582290664,
                "recall_wilson_95": [
                  0.8288805310045245,
                  0.872088915325944
                ],
                "false_positive_rate": 0.2778402699662542,
                "false_positive_wilson_95": [
                  0.2493989905967806,
                  0.3081932380148543
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 885,
                "fp": 247,
                "fn": 154,
                "tn": 642,
                "precision": 0.7818021201413428,
                "recall": 0.8517805582290664,
                "recall_wilson_95": [
                  0.8288805310045245,
                  0.872088915325944
                ],
                "false_positive_rate": 0.2778402699662542,
                "false_positive_wilson_95": [
                  0.2493989905967806,
                  0.3081932380148543
                ],
                "f1": 0.8152924919391985,
                "negative_errors": 0
              },
              "harmful_f1": 0.8152924919391985,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8693378278475258,
                "scored_rows": 1928,
                "excluded_rows": 0,
                "positive_support": 1039,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "Controlled/Regulated Substances": {
                  "support": 112,
                  "tp": 100,
                  "recall": 0.8928571428571429,
                  "recall_wilson_95": [
                    0.8220190653154164,
                    0.9376398762748422
                  ]
                },
                "Copyright/Trademark/Plagiarism": {
                  "support": 4,
                  "tp": 2,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.15003898915214947,
                    0.8499610108478506
                  ]
                },
                "Criminal Planning/Confessions": {
                  "support": 502,
                  "tp": 478,
                  "recall": 0.952191235059761,
                  "recall_wilson_95": [
                    0.9298494365746138,
                    0.9676649764559907
                  ]
                },
                "Fraud/Deception": {
                  "support": 14,
                  "tp": 11,
                  "recall": 0.7857142857142857,
                  "recall_wilson_95": [
                    0.524107694133997,
                    0.9242861328730683
                  ]
                },
                "Guns and Illegal Weapons": {
                  "support": 62,
                  "tp": 57,
                  "recall": 0.9193548387096774,
                  "recall_wilson_95": [
                    0.8247142865138739,
                    0.9650616530140073
                  ]
                },
                "Harassment": {
                  "support": 152,
                  "tp": 133,
                  "recall": 0.875,
                  "recall_wilson_95": [
                    0.8130162507557277,
                    0.9184964087296055
                  ]
                },
                "Hate/Identity Hate": {
                  "support": 175,
                  "tp": 155,
                  "recall": 0.8857142857142857,
                  "recall_wilson_95": [
                    0.8300698699991705,
                    0.9247886560395292
                  ]
                },
                "High Risk Gov Decision Making": {
                  "support": 3,
                  "tp": 3,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.4385029682449546,
                    1.0
                  ]
                },
                "Illegal Activity": {
                  "support": 12,
                  "tp": 12,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.7575059933447592,
                    1.0
                  ]
                },
                "Immoral/Unethical": {
                  "support": 14,
                  "tp": 12,
                  "recall": 0.8571428571428571,
                  "recall_wilson_95": [
                    0.6005862050302965,
                    0.9599060787285351
                  ]
                },
                "Malware": {
                  "support": 8,
                  "tp": 6,
                  "recall": 0.75,
                  "recall_wilson_95": [
                    0.40927543031016883,
                    0.9285207872478909
                  ]
                },
                "Manipulation": {
                  "support": 2,
                  "tp": 1,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.09453120573423074,
                    0.9054687942657693
                  ]
                },
                "Needs Caution": {
                  "support": 69,
                  "tp": 53,
                  "recall": 0.7681159420289855,
                  "recall_wilson_95": [
                    0.6560315295999074,
                    0.8519209551392967
                  ]
                },
                "Other": {
                  "support": 14,
                  "tp": 14,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.7846891972623643,
                    1.0
                  ]
                },
                "PII/Privacy": {
                  "support": 99,
                  "tp": 73,
                  "recall": 0.7373737373737373,
                  "recall_wilson_95": [
                    0.6429956284754338,
                    0.814018503169133
                  ]
                },
                "Political/Misinformation/Conspiracy": {
                  "support": 21,
                  "tp": 15,
                  "recall": 0.7142857142857143,
                  "recall_wilson_95": [
                    0.5004361928391441,
                    0.8618613704787788
                  ]
                },
                "Profanity": {
                  "support": 98,
                  "tp": 80,
                  "recall": 0.8163265306122449,
                  "recall_wilson_95": [
                    0.7282500277598462,
                    0.8805393662448048
                  ]
                },
                "Sexual": {
                  "support": 99,
                  "tp": 89,
                  "recall": 0.898989898989899,
                  "recall_wilson_95": [
                    0.8239693643888216,
                    0.9442033249711951
                  ]
                },
                "Sexual (minor)": {
                  "support": 11,
                  "tp": 10,
                  "recall": 0.9090909090909091,
                  "recall_wilson_95": [
                    0.6226415635484043,
                    0.9837678271141151
                  ]
                },
                "Suicide and Self Harm": {
                  "support": 38,
                  "tp": 31,
                  "recall": 0.8157894736842105,
                  "recall_wilson_95": [
                    0.665811419288686,
                    0.9077823481477872
                  ]
                },
                "Threat": {
                  "support": 16,
                  "tp": 16,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8063923194655637,
                    1.0000000000000002
                  ]
                },
                "Unauthorized Advice": {
                  "support": 23,
                  "tp": 2,
                  "recall": 0.08695652173913043,
                  "recall_wilson_95": [
                    0.024180004484220363,
                    0.26795981075743663
                  ]
                },
                "Violence": {
                  "support": 176,
                  "tp": 167,
                  "recall": 0.9488636363636364,
                  "recall_wilson_95": [
                    0.9056847740581433,
                    0.972866818194728
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_prompt_fine5000.json",
          "input_sha256": "09eedb4a48b3a6eabf03e25ec73af536f0039cb799f2c030d5c2e12683d06323",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1928,
            "correct": 1585,
            "accuracy": 0.8220954356846473,
            "wilson_95": [
              0.8043892901056513,
              0.8385206103569816
            ],
            "macro_f1": 0.8206514059917202,
            "errors": 0,
            "probability_rows": 1928,
            "multiclass_brier": 0.27770013211618255,
            "nll": 0.4497272504978951,
            "ece_10_equal_width": 0.07318495850622406,
            "p50_ms": 32.868375070393085,
            "p95_ms": 79.75445804186165,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1928,
            "per_class": {
              "safe": {
                "support": 889,
                "negative_support": 1039,
                "tp": 706,
                "fp": 160,
                "fn": 183,
                "tn": 879,
                "precision": 0.815242494226328,
                "recall": 0.7941507311586051,
                "recall_wilson_95": [
                  0.7663341385324997,
                  0.8194361511791836
                ],
                "false_positive_rate": 0.15399422521655437,
                "false_positive_wilson_95": [
                  0.13332499774302067,
                  0.17721257830129286
                ]
              },
              "unsafe": {
                "support": 1039,
                "negative_support": 889,
                "tp": 879,
                "fp": 183,
                "fn": 160,
                "tn": 706,
                "precision": 0.827683615819209,
                "recall": 0.8460057747834456,
                "recall_wilson_95": [
                  0.8227874216987072,
                  0.8666750022569794
                ],
                "false_positive_rate": 0.20584926884139482,
                "false_positive_wilson_95": [
                  0.1805638488208164,
                  0.2336658614675002
                ]
              }
            },
            "majority_baseline": 0.5389004149377593,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 879,
              "fp": 183,
              "fn": 160,
              "tn": 706,
              "precision": 0.827683615819209,
              "recall": 0.8460057747834456,
              "recall_wilson_95": [
                0.8227874216987072,
                0.8666750022569794
              ],
              "false_positive_rate": 0.20584926884139482,
              "false_positive_wilson_95": [
                0.1805638488208164,
                0.2336658614675002
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1039,
              "negative_support": 889,
              "tp": 879,
              "fp": 183,
              "fn": 160,
              "tn": 706,
              "precision": 0.827683615819209,
              "recall": 0.8460057747834456,
              "recall_wilson_95": [
                0.8227874216987072,
                0.8666750022569794
              ],
              "false_positive_rate": 0.20584926884139482,
              "false_positive_wilson_95": [
                0.1805638488208164,
                0.2336658614675002
              ],
              "f1": 0.8367444074250358,
              "negative_errors": 0
            },
            "harmful_f1": 0.8367444074250358,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.905319435035095,
              "scored_rows": 1928,
              "excluded_rows": 0,
              "positive_support": 1039,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "Controlled/Regulated Substances": {
                "support": 112,
                "tp": 102,
                "recall": 0.9107142857142857,
                "recall_wilson_95": [
                  0.8434104235612726,
                  0.950778469919452
                ]
              },
              "Copyright/Trademark/Plagiarism": {
                "support": 4,
                "tp": 2,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.15003898915214947,
                  0.8499610108478506
                ]
              },
              "Criminal Planning/Confessions": {
                "support": 502,
                "tp": 473,
                "recall": 0.9422310756972112,
                "recall_wilson_95": [
                  0.9182658053562948,
                  0.9594795677397149
                ]
              },
              "Fraud/Deception": {
                "support": 14,
                "tp": 11,
                "recall": 0.7857142857142857,
                "recall_wilson_95": [
                  0.524107694133997,
                  0.9242861328730683
                ]
              },
              "Guns and Illegal Weapons": {
                "support": 62,
                "tp": 58,
                "recall": 0.9354838709677419,
                "recall_wilson_95": [
                  0.8455248427224711,
                  0.9746270944795592
                ]
              },
              "Harassment": {
                "support": 152,
                "tp": 127,
                "recall": 0.8355263157894737,
                "recall_wilson_95": [
                  0.7684693638513513,
                  0.8860419630565783
                ]
              },
              "Hate/Identity Hate": {
                "support": 175,
                "tp": 142,
                "recall": 0.8114285714285714,
                "recall_wilson_95": [
                  0.7470208730451312,
                  0.8624574924231522
                ]
              },
              "High Risk Gov Decision Making": {
                "support": 3,
                "tp": 3,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.4385029682449546,
                  1.0
                ]
              },
              "Illegal Activity": {
                "support": 12,
                "tp": 12,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.7575059933447592,
                  1.0
                ]
              },
              "Immoral/Unethical": {
                "support": 14,
                "tp": 12,
                "recall": 0.8571428571428571,
                "recall_wilson_95": [
                  0.6005862050302965,
                  0.9599060787285351
                ]
              },
              "Malware": {
                "support": 8,
                "tp": 6,
                "recall": 0.75,
                "recall_wilson_95": [
                  0.40927543031016883,
                  0.9285207872478909
                ]
              },
              "Manipulation": {
                "support": 2,
                "tp": 1,
                "recall": 0.5,
                "recall_wilson_95": [
                  0.09453120573423074,
                  0.9054687942657693
                ]
              },
              "Needs Caution": {
                "support": 69,
                "tp": 52,
                "recall": 0.7536231884057971,
                "recall_wilson_95": [
                  0.6403933718423956,
                  0.8401022218298245
                ]
              },
              "Other": {
                "support": 14,
                "tp": 14,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.7846891972623643,
                  1.0
                ]
              },
              "PII/Privacy": {
                "support": 99,
                "tp": 88,
                "recall": 0.8888888888888888,
                "recall_wilson_95": [
                  0.8119109140413756,
                  0.9368143654614254
                ]
              },
              "Political/Misinformation/Conspiracy": {
                "support": 21,
                "tp": 13,
                "recall": 0.6190476190476191,
                "recall_wilson_95": [
                  0.4087865377511576,
                  0.7924898863143551
                ]
              },
              "Profanity": {
                "support": 98,
                "tp": 78,
                "recall": 0.7959183673469388,
                "recall_wilson_95": [
                  0.7056896702936686,
                  0.8638229886139082
                ]
              },
              "Sexual": {
                "support": 99,
                "tp": 92,
                "recall": 0.9292929292929293,
                "recall_wilson_95": [
                  0.8611851492148981,
                  0.9653297697167653
                ]
              },
              "Sexual (minor)": {
                "support": 11,
                "tp": 9,
                "recall": 0.8181818181818182,
                "recall_wilson_95": [
                  0.523019438039156,
                  0.9486323102539149
                ]
              },
              "Suicide and Self Harm": {
                "support": 38,
                "tp": 29,
                "recall": 0.7631578947368421,
                "recall_wilson_95": [
                  0.607931876367602,
                  0.8700629298294591
                ]
              },
              "Threat": {
                "support": 16,
                "tp": 16,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8063923194655637,
                  1.0000000000000002
                ]
              },
              "Unauthorized Advice": {
                "support": 23,
                "tp": 2,
                "recall": 0.08695652173913043,
                "recall_wilson_95": [
                  0.024180004484220363,
                  0.26795981075743663
                ]
              },
              "Violence": {
                "support": 176,
                "tp": 160,
                "recall": 0.9090909090909091,
                "recall_wilson_95": [
                  0.8574379685932055,
                  0.9432672800423226
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1928,
            "summary": {
              "n": 1928,
              "correct": 1585,
              "accuracy": 0.8220954356846473,
              "wilson_95": [
                0.8043892901056513,
                0.8385206103569816
              ],
              "macro_f1": 0.8206514059917202,
              "errors": 0,
              "probability_rows": 1928,
              "multiclass_brier": 0.27770013211618255,
              "nll": 0.4497272504978951,
              "ece_10_equal_width": 0.07318495850622406,
              "p50_ms": 32.868375070393085,
              "p95_ms": 79.75445804186165,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1928,
              "per_class": {
                "safe": {
                  "support": 889,
                  "negative_support": 1039,
                  "tp": 706,
                  "fp": 160,
                  "fn": 183,
                  "tn": 879,
                  "precision": 0.815242494226328,
                  "recall": 0.7941507311586051,
                  "recall_wilson_95": [
                    0.7663341385324997,
                    0.8194361511791836
                  ],
                  "false_positive_rate": 0.15399422521655437,
                  "false_positive_wilson_95": [
                    0.13332499774302067,
                    0.17721257830129286
                  ]
                },
                "unsafe": {
                  "support": 1039,
                  "negative_support": 889,
                  "tp": 879,
                  "fp": 183,
                  "fn": 160,
                  "tn": 706,
                  "precision": 0.827683615819209,
                  "recall": 0.8460057747834456,
                  "recall_wilson_95": [
                    0.8227874216987072,
                    0.8666750022569794
                  ],
                  "false_positive_rate": 0.20584926884139482,
                  "false_positive_wilson_95": [
                    0.1805638488208164,
                    0.2336658614675002
                  ]
                }
              },
              "majority_baseline": 0.5389004149377593,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 879,
                "fp": 183,
                "fn": 160,
                "tn": 706,
                "precision": 0.827683615819209,
                "recall": 0.8460057747834456,
                "recall_wilson_95": [
                  0.8227874216987072,
                  0.8666750022569794
                ],
                "false_positive_rate": 0.20584926884139482,
                "false_positive_wilson_95": [
                  0.1805638488208164,
                  0.2336658614675002
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1039,
                "negative_support": 889,
                "tp": 879,
                "fp": 183,
                "fn": 160,
                "tn": 706,
                "precision": 0.827683615819209,
                "recall": 0.8460057747834456,
                "recall_wilson_95": [
                  0.8227874216987072,
                  0.8666750022569794
                ],
                "false_positive_rate": 0.20584926884139482,
                "false_positive_wilson_95": [
                  0.1805638488208164,
                  0.2336658614675002
                ],
                "f1": 0.8367444074250358,
                "negative_errors": 0
              },
              "harmful_f1": 0.8367444074250358,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.905319435035095,
                "scored_rows": 1928,
                "excluded_rows": 0,
                "positive_support": 1039,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "Controlled/Regulated Substances": {
                  "support": 112,
                  "tp": 102,
                  "recall": 0.9107142857142857,
                  "recall_wilson_95": [
                    0.8434104235612726,
                    0.950778469919452
                  ]
                },
                "Copyright/Trademark/Plagiarism": {
                  "support": 4,
                  "tp": 2,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.15003898915214947,
                    0.8499610108478506
                  ]
                },
                "Criminal Planning/Confessions": {
                  "support": 502,
                  "tp": 473,
                  "recall": 0.9422310756972112,
                  "recall_wilson_95": [
                    0.9182658053562948,
                    0.9594795677397149
                  ]
                },
                "Fraud/Deception": {
                  "support": 14,
                  "tp": 11,
                  "recall": 0.7857142857142857,
                  "recall_wilson_95": [
                    0.524107694133997,
                    0.9242861328730683
                  ]
                },
                "Guns and Illegal Weapons": {
                  "support": 62,
                  "tp": 58,
                  "recall": 0.9354838709677419,
                  "recall_wilson_95": [
                    0.8455248427224711,
                    0.9746270944795592
                  ]
                },
                "Harassment": {
                  "support": 152,
                  "tp": 127,
                  "recall": 0.8355263157894737,
                  "recall_wilson_95": [
                    0.7684693638513513,
                    0.8860419630565783
                  ]
                },
                "Hate/Identity Hate": {
                  "support": 175,
                  "tp": 142,
                  "recall": 0.8114285714285714,
                  "recall_wilson_95": [
                    0.7470208730451312,
                    0.8624574924231522
                  ]
                },
                "High Risk Gov Decision Making": {
                  "support": 3,
                  "tp": 3,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.4385029682449546,
                    1.0
                  ]
                },
                "Illegal Activity": {
                  "support": 12,
                  "tp": 12,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.7575059933447592,
                    1.0
                  ]
                },
                "Immoral/Unethical": {
                  "support": 14,
                  "tp": 12,
                  "recall": 0.8571428571428571,
                  "recall_wilson_95": [
                    0.6005862050302965,
                    0.9599060787285351
                  ]
                },
                "Malware": {
                  "support": 8,
                  "tp": 6,
                  "recall": 0.75,
                  "recall_wilson_95": [
                    0.40927543031016883,
                    0.9285207872478909
                  ]
                },
                "Manipulation": {
                  "support": 2,
                  "tp": 1,
                  "recall": 0.5,
                  "recall_wilson_95": [
                    0.09453120573423074,
                    0.9054687942657693
                  ]
                },
                "Needs Caution": {
                  "support": 69,
                  "tp": 52,
                  "recall": 0.7536231884057971,
                  "recall_wilson_95": [
                    0.6403933718423956,
                    0.8401022218298245
                  ]
                },
                "Other": {
                  "support": 14,
                  "tp": 14,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.7846891972623643,
                    1.0
                  ]
                },
                "PII/Privacy": {
                  "support": 99,
                  "tp": 88,
                  "recall": 0.8888888888888888,
                  "recall_wilson_95": [
                    0.8119109140413756,
                    0.9368143654614254
                  ]
                },
                "Political/Misinformation/Conspiracy": {
                  "support": 21,
                  "tp": 13,
                  "recall": 0.6190476190476191,
                  "recall_wilson_95": [
                    0.4087865377511576,
                    0.7924898863143551
                  ]
                },
                "Profanity": {
                  "support": 98,
                  "tp": 78,
                  "recall": 0.7959183673469388,
                  "recall_wilson_95": [
                    0.7056896702936686,
                    0.8638229886139082
                  ]
                },
                "Sexual": {
                  "support": 99,
                  "tp": 92,
                  "recall": 0.9292929292929293,
                  "recall_wilson_95": [
                    0.8611851492148981,
                    0.9653297697167653
                  ]
                },
                "Sexual (minor)": {
                  "support": 11,
                  "tp": 9,
                  "recall": 0.8181818181818182,
                  "recall_wilson_95": [
                    0.523019438039156,
                    0.9486323102539149
                  ]
                },
                "Suicide and Self Harm": {
                  "support": 38,
                  "tp": 29,
                  "recall": 0.7631578947368421,
                  "recall_wilson_95": [
                    0.607931876367602,
                    0.8700629298294591
                  ]
                },
                "Threat": {
                  "support": 16,
                  "tp": 16,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8063923194655637,
                    1.0000000000000002
                  ]
                },
                "Unauthorized Advice": {
                  "support": 23,
                  "tp": 2,
                  "recall": 0.08695652173913043,
                  "recall_wilson_95": [
                    0.024180004484220363,
                    0.26795981075743663
                  ]
                },
                "Violence": {
                  "support": 176,
                  "tp": 160,
                  "recall": 0.9090909090909091,
                  "recall_wilson_95": [
                    0.8574379685932055,
                    0.9432672800423226
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": true,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.17842323651452274,
          "harmful_f1_delta_fine_minus_reference": 0.2096810573177057,
          "accuracy_interval": [
            0.1492005147793445,
            0.20913061626523344
          ],
          "harmful_f1_interval": [
            0.17842575303071276,
            0.24217080339059258
          ],
          "confidence": 0.9875,
          "n_rows": 1928,
          "n_text_groups": 1911,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": true,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.20850622406638997,
          "harmful_f1_delta_fine_minus_reference": 0.23113297280354295,
          "accuracy_interval": [
            0.17808930425752856,
            0.238317819415994
          ],
          "harmful_f1_interval": [
            0.19919097100214903,
            0.2621892587981519
          ],
          "confidence": 0.9875,
          "n_rows": 1928,
          "n_text_groups": 1911,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.030082987551867224,
          "harmful_f1_delta_fine_minus_reference": 0.021451915485837247,
          "accuracy_interval": [
            0.016037247801345078,
            0.04406485338435627
          ],
          "harmful_f1_interval": [
            0.008302323281938886,
            0.03426186856586725
          ],
          "confidence": 0.95,
          "n_rows": 1928,
          "n_text_groups": 1911,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "aegis_response",
      "n": 813,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_response_base.json",
          "input_sha256": "f1275223f39be03242201fb22b5059cfe9a45788b5eaff1bb79b13faf13f1ff0",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 813,
            "correct": 475,
            "accuracy": 0.5842558425584256,
            "wilson_95": [
              0.5500590609416522,
              0.6176601439549652
            ],
            "macro_f1": 0.5314200941272764,
            "errors": 0,
            "probability_rows": 813,
            "multiclass_brier": 0.5279505783763837,
            "nll": 0.7390274458683045,
            "ece_10_equal_width": 0.14417404674046744,
            "p50_ms": 55.67575013265014,
            "p95_ms": 254.74674999713898,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 813,
            "per_class": {
              "safe": {
                "support": 419,
                "negative_support": 394,
                "tp": 374,
                "fp": 293,
                "fn": 45,
                "tn": 101,
                "precision": 0.56071964017991,
                "recall": 0.8926014319809069,
                "recall_wilson_95": [
                  0.859308711003355,
                  0.9187606886188624
                ],
                "false_positive_rate": 0.7436548223350253,
                "false_positive_wilson_95": [
                  0.6983343297827348,
                  0.7842699733714363
                ]
              },
              "unsafe": {
                "support": 394,
                "negative_support": 419,
                "tp": 101,
                "fp": 45,
                "fn": 293,
                "tn": 374,
                "precision": 0.6917808219178082,
                "recall": 0.2563451776649746,
                "recall_wilson_95": [
                  0.21573002662856364,
                  0.3016656702172652
                ],
                "false_positive_rate": 0.10739856801909307,
                "false_positive_wilson_95": [
                  0.08123931138113771,
                  0.140691288996645
                ]
              }
            },
            "majority_baseline": 0.5153751537515375,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 101,
              "fp": 45,
              "fn": 293,
              "tn": 374,
              "precision": 0.6917808219178082,
              "recall": 0.2563451776649746,
              "recall_wilson_95": [
                0.21573002662856364,
                0.3016656702172652
              ],
              "false_positive_rate": 0.10739856801909307,
              "false_positive_wilson_95": [
                0.08123931138113771,
                0.140691288996645
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 101,
              "fp": 45,
              "fn": 293,
              "tn": 374,
              "precision": 0.6917808219178082,
              "recall": 0.2563451776649746,
              "recall_wilson_95": [
                0.21573002662856364,
                0.3016656702172652
              ],
              "false_positive_rate": 0.10739856801909307,
              "false_positive_wilson_95": [
                0.08123931138113771,
                0.140691288996645
              ],
              "f1": 0.37407407407407406,
              "negative_errors": 0
            },
            "harmful_f1": 0.37407407407407406,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.6324323253572207,
              "scored_rows": 813,
              "excluded_rows": 0,
              "positive_support": 394,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 813,
            "summary": {
              "n": 813,
              "correct": 475,
              "accuracy": 0.5842558425584256,
              "wilson_95": [
                0.5500590609416522,
                0.6176601439549652
              ],
              "macro_f1": 0.5314200941272764,
              "errors": 0,
              "probability_rows": 813,
              "multiclass_brier": 0.5279505783763837,
              "nll": 0.7390274458683045,
              "ece_10_equal_width": 0.14417404674046744,
              "p50_ms": 55.67575013265014,
              "p95_ms": 254.74674999713898,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 813,
              "per_class": {
                "safe": {
                  "support": 419,
                  "negative_support": 394,
                  "tp": 374,
                  "fp": 293,
                  "fn": 45,
                  "tn": 101,
                  "precision": 0.56071964017991,
                  "recall": 0.8926014319809069,
                  "recall_wilson_95": [
                    0.859308711003355,
                    0.9187606886188624
                  ],
                  "false_positive_rate": 0.7436548223350253,
                  "false_positive_wilson_95": [
                    0.6983343297827348,
                    0.7842699733714363
                  ]
                },
                "unsafe": {
                  "support": 394,
                  "negative_support": 419,
                  "tp": 101,
                  "fp": 45,
                  "fn": 293,
                  "tn": 374,
                  "precision": 0.6917808219178082,
                  "recall": 0.2563451776649746,
                  "recall_wilson_95": [
                    0.21573002662856364,
                    0.3016656702172652
                  ],
                  "false_positive_rate": 0.10739856801909307,
                  "false_positive_wilson_95": [
                    0.08123931138113771,
                    0.140691288996645
                  ]
                }
              },
              "majority_baseline": 0.5153751537515375,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 101,
                "fp": 45,
                "fn": 293,
                "tn": 374,
                "precision": 0.6917808219178082,
                "recall": 0.2563451776649746,
                "recall_wilson_95": [
                  0.21573002662856364,
                  0.3016656702172652
                ],
                "false_positive_rate": 0.10739856801909307,
                "false_positive_wilson_95": [
                  0.08123931138113771,
                  0.140691288996645
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 101,
                "fp": 45,
                "fn": 293,
                "tn": 374,
                "precision": 0.6917808219178082,
                "recall": 0.2563451776649746,
                "recall_wilson_95": [
                  0.21573002662856364,
                  0.3016656702172652
                ],
                "false_positive_rate": 0.10739856801909307,
                "false_positive_wilson_95": [
                  0.08123931138113771,
                  0.140691288996645
                ],
                "f1": 0.37407407407407406,
                "negative_errors": 0
              },
              "harmful_f1": 0.37407407407407406,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.6324323253572207,
                "scored_rows": 813,
                "excluded_rows": 0,
                "positive_support": 394,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_response_fine1000.json",
          "input_sha256": "91281fadaeffccf4c86f10f48fea566fd4c7b5d061fc1e2f20b52059ff65fcb9",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 813,
            "correct": 597,
            "accuracy": 0.7343173431734318,
            "wilson_95": [
              0.7029051139564306,
              0.7635256675443818
            ],
            "macro_f1": 0.7298105682951146,
            "errors": 0,
            "probability_rows": 813,
            "multiclass_brier": 0.36591246388683885,
            "nll": 0.5498282351261122,
            "ece_10_equal_width": 0.10348228782287826,
            "p50_ms": 60.90633315034211,
            "p95_ms": 68.87891702353954,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 813,
            "per_class": {
              "safe": {
                "support": 419,
                "negative_support": 394,
                "tp": 351,
                "fp": 148,
                "fn": 68,
                "tn": 246,
                "precision": 0.7034068136272545,
                "recall": 0.837708830548926,
                "recall_wilson_95": [
                  0.7993629558138261,
                  0.8699186250162271
                ],
                "false_positive_rate": 0.3756345177664975,
                "false_positive_wilson_95": [
                  0.329232451523448,
                  0.42443826874161045
                ]
              },
              "unsafe": {
                "support": 394,
                "negative_support": 419,
                "tp": 246,
                "fp": 68,
                "fn": 148,
                "tn": 351,
                "precision": 0.7834394904458599,
                "recall": 0.6243654822335025,
                "recall_wilson_95": [
                  0.5755617312583895,
                  0.670767548476552
                ],
                "false_positive_rate": 0.162291169451074,
                "false_positive_wilson_95": [
                  0.13008137498377287,
                  0.20063704418617395
                ]
              }
            },
            "majority_baseline": 0.5153751537515375,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 246,
              "fp": 68,
              "fn": 148,
              "tn": 351,
              "precision": 0.7834394904458599,
              "recall": 0.6243654822335025,
              "recall_wilson_95": [
                0.5755617312583895,
                0.670767548476552
              ],
              "false_positive_rate": 0.162291169451074,
              "false_positive_wilson_95": [
                0.13008137498377287,
                0.20063704418617395
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 246,
              "fp": 68,
              "fn": 148,
              "tn": 351,
              "precision": 0.7834394904458599,
              "recall": 0.6243654822335025,
              "recall_wilson_95": [
                0.5755617312583895,
                0.670767548476552
              ],
              "false_positive_rate": 0.162291169451074,
              "false_positive_wilson_95": [
                0.13008137498377287,
                0.20063704418617395
              ],
              "f1": 0.6949152542372882,
              "negative_errors": 0
            },
            "harmful_f1": 0.6949152542372882,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8257169869753955,
              "scored_rows": 813,
              "excluded_rows": 0,
              "positive_support": 394,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 813,
            "summary": {
              "n": 813,
              "correct": 597,
              "accuracy": 0.7343173431734318,
              "wilson_95": [
                0.7029051139564306,
                0.7635256675443818
              ],
              "macro_f1": 0.7298105682951146,
              "errors": 0,
              "probability_rows": 813,
              "multiclass_brier": 0.36591246388683885,
              "nll": 0.5498282351261122,
              "ece_10_equal_width": 0.10348228782287826,
              "p50_ms": 60.90633315034211,
              "p95_ms": 68.87891702353954,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 813,
              "per_class": {
                "safe": {
                  "support": 419,
                  "negative_support": 394,
                  "tp": 351,
                  "fp": 148,
                  "fn": 68,
                  "tn": 246,
                  "precision": 0.7034068136272545,
                  "recall": 0.837708830548926,
                  "recall_wilson_95": [
                    0.7993629558138261,
                    0.8699186250162271
                  ],
                  "false_positive_rate": 0.3756345177664975,
                  "false_positive_wilson_95": [
                    0.329232451523448,
                    0.42443826874161045
                  ]
                },
                "unsafe": {
                  "support": 394,
                  "negative_support": 419,
                  "tp": 246,
                  "fp": 68,
                  "fn": 148,
                  "tn": 351,
                  "precision": 0.7834394904458599,
                  "recall": 0.6243654822335025,
                  "recall_wilson_95": [
                    0.5755617312583895,
                    0.670767548476552
                  ],
                  "false_positive_rate": 0.162291169451074,
                  "false_positive_wilson_95": [
                    0.13008137498377287,
                    0.20063704418617395
                  ]
                }
              },
              "majority_baseline": 0.5153751537515375,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 246,
                "fp": 68,
                "fn": 148,
                "tn": 351,
                "precision": 0.7834394904458599,
                "recall": 0.6243654822335025,
                "recall_wilson_95": [
                  0.5755617312583895,
                  0.670767548476552
                ],
                "false_positive_rate": 0.162291169451074,
                "false_positive_wilson_95": [
                  0.13008137498377287,
                  0.20063704418617395
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 246,
                "fp": 68,
                "fn": 148,
                "tn": 351,
                "precision": 0.7834394904458599,
                "recall": 0.6243654822335025,
                "recall_wilson_95": [
                  0.5755617312583895,
                  0.670767548476552
                ],
                "false_positive_rate": 0.162291169451074,
                "false_positive_wilson_95": [
                  0.13008137498377287,
                  0.20063704418617395
                ],
                "f1": 0.6949152542372882,
                "negative_errors": 0
              },
              "harmful_f1": 0.6949152542372882,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8257169869753955,
                "scored_rows": 813,
                "excluded_rows": 0,
                "positive_support": 394,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/aegis_response_fine5000.json",
          "input_sha256": "467c69e9603b3f6fffe25a5a46ad9b7561e0008c5349c6f5ccbda522140588ac",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 813,
            "correct": 642,
            "accuracy": 0.7896678966789668,
            "wilson_95": [
              0.7603242357480006,
              0.8162870453199328
            ],
            "macro_f1": 0.7881860911563279,
            "errors": 0,
            "probability_rows": 813,
            "multiclass_brier": 0.31570323079950796,
            "nll": 0.493479481818412,
            "ece_10_equal_width": 0.10184895448954487,
            "p50_ms": 60.04987517371774,
            "p95_ms": 62.789250165224075,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 813,
            "per_class": {
              "safe": {
                "support": 419,
                "negative_support": 394,
                "tp": 355,
                "fp": 107,
                "fn": 64,
                "tn": 287,
                "precision": 0.7683982683982684,
                "recall": 0.847255369928401,
                "recall_wilson_95": [
                  0.8096672009955875,
                  0.8785340004939727
                ],
                "false_positive_rate": 0.2715736040609137,
                "false_positive_wilson_95": [
                  0.23001875782517506,
                  0.31753970796778935
                ]
              },
              "unsafe": {
                "support": 394,
                "negative_support": 419,
                "tp": 287,
                "fp": 64,
                "fn": 107,
                "tn": 355,
                "precision": 0.8176638176638177,
                "recall": 0.7284263959390863,
                "recall_wilson_95": [
                  0.6824602920322107,
                  0.7699812421748249
                ],
                "false_positive_rate": 0.15274463007159905,
                "false_positive_wilson_95": [
                  0.1214659995060274,
                  0.1903327990044126
                ]
              }
            },
            "majority_baseline": 0.5153751537515375,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 287,
              "fp": 64,
              "fn": 107,
              "tn": 355,
              "precision": 0.8176638176638177,
              "recall": 0.7284263959390863,
              "recall_wilson_95": [
                0.6824602920322107,
                0.7699812421748249
              ],
              "false_positive_rate": 0.15274463007159905,
              "false_positive_wilson_95": [
                0.1214659995060274,
                0.1903327990044126
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 394,
              "negative_support": 419,
              "tp": 287,
              "fp": 64,
              "fn": 107,
              "tn": 355,
              "precision": 0.8176638176638177,
              "recall": 0.7284263959390863,
              "recall_wilson_95": [
                0.6824602920322107,
                0.7699812421748249
              ],
              "false_positive_rate": 0.15274463007159905,
              "false_positive_wilson_95": [
                0.1214659995060274,
                0.1903327990044126
              ],
              "f1": 0.7704697986577181,
              "negative_errors": 0
            },
            "harmful_f1": 0.7704697986577181,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8819097051580975,
              "scored_rows": 813,
              "excluded_rows": 0,
              "positive_support": 394,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 813,
            "summary": {
              "n": 813,
              "correct": 642,
              "accuracy": 0.7896678966789668,
              "wilson_95": [
                0.7603242357480006,
                0.8162870453199328
              ],
              "macro_f1": 0.7881860911563279,
              "errors": 0,
              "probability_rows": 813,
              "multiclass_brier": 0.31570323079950796,
              "nll": 0.493479481818412,
              "ece_10_equal_width": 0.10184895448954487,
              "p50_ms": 60.04987517371774,
              "p95_ms": 62.789250165224075,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 813,
              "per_class": {
                "safe": {
                  "support": 419,
                  "negative_support": 394,
                  "tp": 355,
                  "fp": 107,
                  "fn": 64,
                  "tn": 287,
                  "precision": 0.7683982683982684,
                  "recall": 0.847255369928401,
                  "recall_wilson_95": [
                    0.8096672009955875,
                    0.8785340004939727
                  ],
                  "false_positive_rate": 0.2715736040609137,
                  "false_positive_wilson_95": [
                    0.23001875782517506,
                    0.31753970796778935
                  ]
                },
                "unsafe": {
                  "support": 394,
                  "negative_support": 419,
                  "tp": 287,
                  "fp": 64,
                  "fn": 107,
                  "tn": 355,
                  "precision": 0.8176638176638177,
                  "recall": 0.7284263959390863,
                  "recall_wilson_95": [
                    0.6824602920322107,
                    0.7699812421748249
                  ],
                  "false_positive_rate": 0.15274463007159905,
                  "false_positive_wilson_95": [
                    0.1214659995060274,
                    0.1903327990044126
                  ]
                }
              },
              "majority_baseline": 0.5153751537515375,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 287,
                "fp": 64,
                "fn": 107,
                "tn": 355,
                "precision": 0.8176638176638177,
                "recall": 0.7284263959390863,
                "recall_wilson_95": [
                  0.6824602920322107,
                  0.7699812421748249
                ],
                "false_positive_rate": 0.15274463007159905,
                "false_positive_wilson_95": [
                  0.1214659995060274,
                  0.1903327990044126
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 394,
                "negative_support": 419,
                "tp": 287,
                "fp": 64,
                "fn": 107,
                "tn": 355,
                "precision": 0.8176638176638177,
                "recall": 0.7284263959390863,
                "recall_wilson_95": [
                  0.6824602920322107,
                  0.7699812421748249
                ],
                "false_positive_rate": 0.15274463007159905,
                "false_positive_wilson_95": [
                  0.1214659995060274,
                  0.1903327990044126
                ],
                "f1": 0.7704697986577181,
                "negative_errors": 0
              },
              "harmful_f1": 0.7704697986577181,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8819097051580975,
                "scored_rows": 813,
                "excluded_rows": 0,
                "positive_support": 394,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": true,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.15006150061500623,
          "harmful_f1_delta_fine_minus_reference": 0.3208411801632141,
          "accuracy_interval": [
            0.10571453534815596,
            0.19865182726888106
          ],
          "harmful_f1_interval": [
            0.25333296655001525,
            0.39276692978921834
          ],
          "confidence": 0.9875,
          "n_rows": 813,
          "n_text_groups": 811,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": true,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.2054120541205412,
          "harmful_f1_delta_fine_minus_reference": 0.3963957245836441,
          "accuracy_interval": [
            0.15659679408138097,
            0.2548314159612185
          ],
          "harmful_f1_interval": [
            0.32342027003656965,
            0.47070706561984593
          ],
          "confidence": 0.9875,
          "n_rows": 813,
          "n_text_groups": 811,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.05535055350553497,
          "harmful_f1_delta_fine_minus_reference": 0.07555454442042997,
          "accuracy_interval": [
            0.029593094944513006,
            0.07995326104738879
          ],
          "harmful_f1_interval": [
            0.04483707116159,
            0.10712704519080622
          ],
          "confidence": 0.95,
          "n_rows": 813,
          "n_text_groups": 811,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "toxicchat",
      "n": 2853,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/toxicchat_base.json",
          "input_sha256": "060d3c8584b344dbd04353feec368afc0af260c034f11846d18669215f94ab63",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 2853,
            "correct": 2030,
            "accuracy": 0.7115317209954434,
            "wilson_95": [
              0.6946317301946091,
              0.7278628385093365
            ],
            "macro_f1": 0.5889498787254398,
            "errors": 0,
            "probability_rows": 2853,
            "multiclass_brier": 0.401486842078352,
            "nll": 0.5959570569762147,
            "ece_10_equal_width": 0.057128106140442225,
            "p50_ms": 29.586165910586715,
            "p95_ms": 162.69887518137693,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 2853,
            "per_class": {
              "safe": {
                "support": 2491,
                "negative_support": 362,
                "tp": 1794,
                "fp": 126,
                "fn": 697,
                "tn": 236,
                "precision": 0.934375,
                "recall": 0.7201926936973103,
                "recall_wilson_95": [
                  0.7022354743385553,
                  0.7374718249455241
                ],
                "false_positive_rate": 0.34806629834254144,
                "false_positive_wilson_95": [
                  0.30082272087487166,
                  0.39850058612650036
                ]
              },
              "unsafe": {
                "support": 362,
                "negative_support": 2491,
                "tp": 236,
                "fp": 697,
                "fn": 126,
                "tn": 1794,
                "precision": 0.2529474812433012,
                "recall": 0.6519337016574586,
                "recall_wilson_95": [
                  0.6014994138734995,
                  0.6991772791251283
                ],
                "false_positive_rate": 0.2798073063026897,
                "false_positive_wilson_95": [
                  0.26252817505447595,
                  0.2977645256614448
                ]
              }
            },
            "majority_baseline": 0.8731160182264284,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 236,
              "fp": 697,
              "fn": 126,
              "tn": 1794,
              "precision": 0.2529474812433012,
              "recall": 0.6519337016574586,
              "recall_wilson_95": [
                0.6014994138734995,
                0.6991772791251283
              ],
              "false_positive_rate": 0.2798073063026897,
              "false_positive_wilson_95": [
                0.26252817505447595,
                0.2977645256614448
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 236,
              "fp": 697,
              "fn": 126,
              "tn": 1794,
              "precision": 0.2529474812433012,
              "recall": 0.6519337016574586,
              "recall_wilson_95": [
                0.6014994138734995,
                0.6991772791251283
              ],
              "false_positive_rate": 0.2798073063026897,
              "false_positive_wilson_95": [
                0.26252817505447595,
                0.2977645256614448
              ],
              "f1": 0.3644787644787645,
              "negative_errors": 0
            },
            "harmful_f1": 0.3644787644787645,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.3376446415529083,
              "scored_rows": 2853,
              "excluded_rows": 0,
              "positive_support": 362,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 2853,
            "summary": {
              "n": 2853,
              "correct": 2030,
              "accuracy": 0.7115317209954434,
              "wilson_95": [
                0.6946317301946091,
                0.7278628385093365
              ],
              "macro_f1": 0.5889498787254398,
              "errors": 0,
              "probability_rows": 2853,
              "multiclass_brier": 0.401486842078352,
              "nll": 0.5959570569762147,
              "ece_10_equal_width": 0.057128106140442225,
              "p50_ms": 29.586165910586715,
              "p95_ms": 162.69887518137693,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 2853,
              "per_class": {
                "safe": {
                  "support": 2491,
                  "negative_support": 362,
                  "tp": 1794,
                  "fp": 126,
                  "fn": 697,
                  "tn": 236,
                  "precision": 0.934375,
                  "recall": 0.7201926936973103,
                  "recall_wilson_95": [
                    0.7022354743385553,
                    0.7374718249455241
                  ],
                  "false_positive_rate": 0.34806629834254144,
                  "false_positive_wilson_95": [
                    0.30082272087487166,
                    0.39850058612650036
                  ]
                },
                "unsafe": {
                  "support": 362,
                  "negative_support": 2491,
                  "tp": 236,
                  "fp": 697,
                  "fn": 126,
                  "tn": 1794,
                  "precision": 0.2529474812433012,
                  "recall": 0.6519337016574586,
                  "recall_wilson_95": [
                    0.6014994138734995,
                    0.6991772791251283
                  ],
                  "false_positive_rate": 0.2798073063026897,
                  "false_positive_wilson_95": [
                    0.26252817505447595,
                    0.2977645256614448
                  ]
                }
              },
              "majority_baseline": 0.8731160182264284,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 236,
                "fp": 697,
                "fn": 126,
                "tn": 1794,
                "precision": 0.2529474812433012,
                "recall": 0.6519337016574586,
                "recall_wilson_95": [
                  0.6014994138734995,
                  0.6991772791251283
                ],
                "false_positive_rate": 0.2798073063026897,
                "false_positive_wilson_95": [
                  0.26252817505447595,
                  0.2977645256614448
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 236,
                "fp": 697,
                "fn": 126,
                "tn": 1794,
                "precision": 0.2529474812433012,
                "recall": 0.6519337016574586,
                "recall_wilson_95": [
                  0.6014994138734995,
                  0.6991772791251283
                ],
                "false_positive_rate": 0.2798073063026897,
                "false_positive_wilson_95": [
                  0.26252817505447595,
                  0.2977645256614448
                ],
                "f1": 0.3644787644787645,
                "negative_errors": 0
              },
              "harmful_f1": 0.3644787644787645,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.3376446415529083,
                "scored_rows": 2853,
                "excluded_rows": 0,
                "positive_support": 362,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/toxicchat_fine1000.json",
          "input_sha256": "b54e4f49b2d3aa1d9ac84471ffd719abf8249687be2f047eeab77cff22874857",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 2853,
            "correct": 2378,
            "accuracy": 0.8335085874518051,
            "wilson_95": [
              0.8193926090308941,
              0.8467276596050699
            ],
            "macro_f1": 0.7023145950474621,
            "errors": 0,
            "probability_rows": 2853,
            "multiclass_brier": 0.24417236083041555,
            "nll": 0.3951675219141415,
            "ece_10_equal_width": 0.019973327388607097,
            "p50_ms": 41.39541694894433,
            "p95_ms": 77.29633385315537,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 2853,
            "per_class": {
              "safe": {
                "support": 2491,
                "negative_support": 362,
                "tp": 2136,
                "fp": 120,
                "fn": 355,
                "tn": 242,
                "precision": 0.9468085106382979,
                "recall": 0.8574869530309113,
                "recall_wilson_95": [
                  0.8432081990140811,
                  0.870664818328622
                ],
                "false_positive_rate": 0.3314917127071823,
                "false_positive_wilson_95": [
                  0.28499043626769616,
                  0.38153177695200735
                ]
              },
              "unsafe": {
                "support": 362,
                "negative_support": 2491,
                "tp": 242,
                "fp": 355,
                "fn": 120,
                "tn": 2136,
                "precision": 0.4053601340033501,
                "recall": 0.6685082872928176,
                "recall_wilson_95": [
                  0.6184682230479925,
                  0.7150095637323037
                ],
                "false_positive_rate": 0.14251304696908873,
                "false_positive_wilson_95": [
                  0.12933518167137803,
                  0.15679180098591894
                ]
              }
            },
            "majority_baseline": 0.8731160182264284,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 242,
              "fp": 355,
              "fn": 120,
              "tn": 2136,
              "precision": 0.4053601340033501,
              "recall": 0.6685082872928176,
              "recall_wilson_95": [
                0.6184682230479925,
                0.7150095637323037
              ],
              "false_positive_rate": 0.14251304696908873,
              "false_positive_wilson_95": [
                0.12933518167137803,
                0.15679180098591894
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 242,
              "fp": 355,
              "fn": 120,
              "tn": 2136,
              "precision": 0.4053601340033501,
              "recall": 0.6685082872928176,
              "recall_wilson_95": [
                0.6184682230479925,
                0.7150095637323037
              ],
              "false_positive_rate": 0.14251304696908873,
              "false_positive_wilson_95": [
                0.12933518167137803,
                0.15679180098591894
              ],
              "f1": 0.5046923879040668,
              "negative_errors": 0
            },
            "harmful_f1": 0.5046923879040668,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.48820156335868187,
              "scored_rows": 2853,
              "excluded_rows": 0,
              "positive_support": 362,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 2853,
            "summary": {
              "n": 2853,
              "correct": 2378,
              "accuracy": 0.8335085874518051,
              "wilson_95": [
                0.8193926090308941,
                0.8467276596050699
              ],
              "macro_f1": 0.7023145950474621,
              "errors": 0,
              "probability_rows": 2853,
              "multiclass_brier": 0.24417236083041555,
              "nll": 0.3951675219141415,
              "ece_10_equal_width": 0.019973327388607097,
              "p50_ms": 41.39541694894433,
              "p95_ms": 77.29633385315537,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 2853,
              "per_class": {
                "safe": {
                  "support": 2491,
                  "negative_support": 362,
                  "tp": 2136,
                  "fp": 120,
                  "fn": 355,
                  "tn": 242,
                  "precision": 0.9468085106382979,
                  "recall": 0.8574869530309113,
                  "recall_wilson_95": [
                    0.8432081990140811,
                    0.870664818328622
                  ],
                  "false_positive_rate": 0.3314917127071823,
                  "false_positive_wilson_95": [
                    0.28499043626769616,
                    0.38153177695200735
                  ]
                },
                "unsafe": {
                  "support": 362,
                  "negative_support": 2491,
                  "tp": 242,
                  "fp": 355,
                  "fn": 120,
                  "tn": 2136,
                  "precision": 0.4053601340033501,
                  "recall": 0.6685082872928176,
                  "recall_wilson_95": [
                    0.6184682230479925,
                    0.7150095637323037
                  ],
                  "false_positive_rate": 0.14251304696908873,
                  "false_positive_wilson_95": [
                    0.12933518167137803,
                    0.15679180098591894
                  ]
                }
              },
              "majority_baseline": 0.8731160182264284,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 242,
                "fp": 355,
                "fn": 120,
                "tn": 2136,
                "precision": 0.4053601340033501,
                "recall": 0.6685082872928176,
                "recall_wilson_95": [
                  0.6184682230479925,
                  0.7150095637323037
                ],
                "false_positive_rate": 0.14251304696908873,
                "false_positive_wilson_95": [
                  0.12933518167137803,
                  0.15679180098591894
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 242,
                "fp": 355,
                "fn": 120,
                "tn": 2136,
                "precision": 0.4053601340033501,
                "recall": 0.6685082872928176,
                "recall_wilson_95": [
                  0.6184682230479925,
                  0.7150095637323037
                ],
                "false_positive_rate": 0.14251304696908873,
                "false_positive_wilson_95": [
                  0.12933518167137803,
                  0.15679180098591894
                ],
                "f1": 0.5046923879040668,
                "negative_errors": 0
              },
              "harmful_f1": 0.5046923879040668,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.48820156335868187,
                "scored_rows": 2853,
                "excluded_rows": 0,
                "positive_support": 362,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/toxicchat_fine5000.json",
          "input_sha256": "ef089061e60cdba9d3f86a01875277959146bc41fba13b4df92e3abd5cb3b51e",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 2853,
            "correct": 2428,
            "accuracy": 0.8510339992989835,
            "wilson_95": [
              0.83749709287895,
              0.863626868275982
            ],
            "macro_f1": 0.7224228494509809,
            "errors": 0,
            "probability_rows": 2853,
            "multiclass_brier": 0.22586918194902073,
            "nll": 0.3736900924535802,
            "ece_10_equal_width": 0.02652369256904652,
            "p50_ms": 32.0013330783695,
            "p95_ms": 61.80595816113055,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 2853,
            "per_class": {
              "safe": {
                "support": 2491,
                "negative_support": 362,
                "tp": 2185,
                "fp": 119,
                "fn": 306,
                "tn": 243,
                "precision": 0.9483506944444444,
                "recall": 0.8771577679646728,
                "recall_wilson_95": [
                  0.8636832548656341,
                  0.889470815649211
                ],
                "false_positive_rate": 0.3287292817679558,
                "false_positive_wilson_95": [
                  0.2823581344957796,
                  0.3786972297603125
                ]
              },
              "unsafe": {
                "support": 362,
                "negative_support": 2491,
                "tp": 243,
                "fp": 306,
                "fn": 119,
                "tn": 2185,
                "precision": 0.4426229508196721,
                "recall": 0.6712707182320442,
                "recall_wilson_95": [
                  0.6213027702396874,
                  0.7176418655042203
                ],
                "false_positive_rate": 0.12284223203532718,
                "false_positive_wilson_95": [
                  0.110529184350789,
                  0.13631674513436598
                ]
              }
            },
            "majority_baseline": 0.8731160182264284,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 243,
              "fp": 306,
              "fn": 119,
              "tn": 2185,
              "precision": 0.4426229508196721,
              "recall": 0.6712707182320442,
              "recall_wilson_95": [
                0.6213027702396874,
                0.7176418655042203
              ],
              "false_positive_rate": 0.12284223203532718,
              "false_positive_wilson_95": [
                0.110529184350789,
                0.13631674513436598
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 362,
              "negative_support": 2491,
              "tp": 243,
              "fp": 306,
              "fn": 119,
              "tn": 2185,
              "precision": 0.4426229508196721,
              "recall": 0.6712707182320442,
              "recall_wilson_95": [
                0.6213027702396874,
                0.7176418655042203
              ],
              "false_positive_rate": 0.12284223203532718,
              "false_positive_wilson_95": [
                0.110529184350789,
                0.13631674513436598
              ],
              "f1": 0.5334796926454446,
              "negative_errors": 0
            },
            "harmful_f1": 0.5334796926454446,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.5636041484942792,
              "scored_rows": 2853,
              "excluded_rows": 0,
              "positive_support": 362,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 2853,
            "summary": {
              "n": 2853,
              "correct": 2428,
              "accuracy": 0.8510339992989835,
              "wilson_95": [
                0.83749709287895,
                0.863626868275982
              ],
              "macro_f1": 0.7224228494509809,
              "errors": 0,
              "probability_rows": 2853,
              "multiclass_brier": 0.22586918194902073,
              "nll": 0.3736900924535802,
              "ece_10_equal_width": 0.02652369256904652,
              "p50_ms": 32.0013330783695,
              "p95_ms": 61.80595816113055,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 2853,
              "per_class": {
                "safe": {
                  "support": 2491,
                  "negative_support": 362,
                  "tp": 2185,
                  "fp": 119,
                  "fn": 306,
                  "tn": 243,
                  "precision": 0.9483506944444444,
                  "recall": 0.8771577679646728,
                  "recall_wilson_95": [
                    0.8636832548656341,
                    0.889470815649211
                  ],
                  "false_positive_rate": 0.3287292817679558,
                  "false_positive_wilson_95": [
                    0.2823581344957796,
                    0.3786972297603125
                  ]
                },
                "unsafe": {
                  "support": 362,
                  "negative_support": 2491,
                  "tp": 243,
                  "fp": 306,
                  "fn": 119,
                  "tn": 2185,
                  "precision": 0.4426229508196721,
                  "recall": 0.6712707182320442,
                  "recall_wilson_95": [
                    0.6213027702396874,
                    0.7176418655042203
                  ],
                  "false_positive_rate": 0.12284223203532718,
                  "false_positive_wilson_95": [
                    0.110529184350789,
                    0.13631674513436598
                  ]
                }
              },
              "majority_baseline": 0.8731160182264284,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 243,
                "fp": 306,
                "fn": 119,
                "tn": 2185,
                "precision": 0.4426229508196721,
                "recall": 0.6712707182320442,
                "recall_wilson_95": [
                  0.6213027702396874,
                  0.7176418655042203
                ],
                "false_positive_rate": 0.12284223203532718,
                "false_positive_wilson_95": [
                  0.110529184350789,
                  0.13631674513436598
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 362,
                "negative_support": 2491,
                "tp": 243,
                "fp": 306,
                "fn": 119,
                "tn": 2185,
                "precision": 0.4426229508196721,
                "recall": 0.6712707182320442,
                "recall_wilson_95": [
                  0.6213027702396874,
                  0.7176418655042203
                ],
                "false_positive_rate": 0.12284223203532718,
                "false_positive_wilson_95": [
                  0.110529184350789,
                  0.13631674513436598
                ],
                "f1": 0.5334796926454446,
                "negative_errors": 0
              },
              "harmful_f1": 0.5334796926454446,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.5636041484942792,
                "scored_rows": 2853,
                "excluded_rows": 0,
                "positive_support": 362,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.12197686645636174,
          "harmful_f1_delta_fine_minus_reference": 0.1402136234253023,
          "accuracy_interval": [
            0.10500428244378057,
            0.1388986320589266
          ],
          "harmful_f1_interval": [
            0.10845339272392411,
            0.17098647613419593
          ],
          "confidence": 0.95,
          "n_rows": 2853,
          "n_text_groups": 2774,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.13950227830354012,
          "harmful_f1_delta_fine_minus_reference": 0.1690009281666801,
          "accuracy_interval": [
            0.12195057676156523,
            0.15719348002358846
          ],
          "harmful_f1_interval": [
            0.13372556979326602,
            0.2032955655760494
          ],
          "confidence": 0.95,
          "n_rows": 2853,
          "n_text_groups": 2774,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.017525411847178374,
          "harmful_f1_delta_fine_minus_reference": 0.028787304741377806,
          "accuracy_interval": [
            0.006295852625863105,
            0.028896771832818326
          ],
          "harmful_f1_interval": [
            0.0027005287140814565,
            0.05444721107604559
          ],
          "confidence": 0.95,
          "n_rows": 2853,
          "n_text_groups": 2774,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "beavertails",
      "n": 3021,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/beavertails_base.json",
          "input_sha256": "9338773ef5661fcebe17d38de7a06b231271f0c0ef7e8c258748b1229d4b46cd",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 3021,
            "correct": 1304,
            "accuracy": 0.43164515061238,
            "wilson_95": [
              0.41408071791152434,
              0.4493832005706635
            ],
            "macro_f1": 0.3264266527979867,
            "errors": 0,
            "probability_rows": 3021,
            "multiclass_brier": 0.8814790772591856,
            "nll": 1.307393055712035,
            "ece_10_equal_width": 0.4396135716650116,
            "p50_ms": 42.67820809036493,
            "p95_ms": 214.3910420127213,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 3021,
            "per_class": {
              "safe": {
                "support": 1288,
                "negative_support": 1733,
                "tp": 1249,
                "fp": 1678,
                "fn": 39,
                "tn": 55,
                "precision": 0.42671677485480014,
                "recall": 0.96972049689441,
                "recall_wilson_95": [
                  0.9588757242210745,
                  0.9777717194246759
                ],
                "false_positive_rate": 0.9682631275245239,
                "false_positive_wilson_95": [
                  0.9589184804554151,
                  0.9755364127977711
                ]
              },
              "unsafe": {
                "support": 1733,
                "negative_support": 1288,
                "tp": 55,
                "fp": 39,
                "fn": 1678,
                "tn": 1249,
                "precision": 0.5851063829787234,
                "recall": 0.031736872475476054,
                "recall_wilson_95": [
                  0.024463587202228878,
                  0.04108151954458489
                ],
                "false_positive_rate": 0.030279503105590064,
                "false_positive_wilson_95": [
                  0.022228280575324118,
                  0.04112427577892556
                ]
              }
            },
            "majority_baseline": 0.5736511089043363,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 55,
              "fp": 39,
              "fn": 1678,
              "tn": 1249,
              "precision": 0.5851063829787234,
              "recall": 0.031736872475476054,
              "recall_wilson_95": [
                0.024463587202228878,
                0.04108151954458489
              ],
              "false_positive_rate": 0.030279503105590064,
              "false_positive_wilson_95": [
                0.022228280575324118,
                0.04112427577892556
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 55,
              "fp": 39,
              "fn": 1678,
              "tn": 1249,
              "precision": 0.5851063829787234,
              "recall": 0.031736872475476054,
              "recall_wilson_95": [
                0.024463587202228878,
                0.04108151954458489
              ],
              "false_positive_rate": 0.030279503105590064,
              "false_positive_wilson_95": [
                0.022228280575324118,
                0.04112427577892556
              ],
              "f1": 0.060207991242474,
              "negative_errors": 0
            },
            "harmful_f1": 0.060207991242474,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.6312380584550155,
              "scored_rows": 3021,
              "excluded_rows": 0,
              "positive_support": 1733,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "animal_abuse": {
                "support": 44,
                "tp": 2,
                "recall": 0.045454545454545456,
                "recall_wilson_95": [
                  0.012555109949594967,
                  0.15134998437629657
                ]
              },
              "child_abuse": {
                "support": 27,
                "tp": 0,
                "recall": 0.0,
                "recall_wilson_95": [
                  0.0,
                  0.12455502974186705
                ]
              },
              "controversial_topics,politics": {
                "support": 95,
                "tp": 3,
                "recall": 0.031578947368421054,
                "recall_wilson_95": [
                  0.010797139474424379,
                  0.08877098647308188
                ]
              },
              "discrimination,stereotype,injustice": {
                "support": 297,
                "tp": 7,
                "recall": 0.02356902356902357,
                "recall_wilson_95": [
                  0.0114627442740084,
                  0.04784244222067581
                ]
              },
              "drug_abuse,weapons,banned_substance": {
                "support": 151,
                "tp": 2,
                "recall": 0.013245033112582781,
                "recall_wilson_95": [
                  0.0036398128620662405,
                  0.0470020428813268
                ]
              },
              "financial_crime,property_crime,theft": {
                "support": 262,
                "tp": 5,
                "recall": 0.019083969465648856,
                "recall_wilson_95": [
                  0.008178399215518155,
                  0.04388819296350676
                ]
              },
              "hate_speech,offensive_language": {
                "support": 302,
                "tp": 26,
                "recall": 0.08609271523178808,
                "recall_wilson_95": [
                  0.05942803805385782,
                  0.12315498726918161
                ]
              },
              "misinformation_regarding_ethics,laws_and_safety": {
                "support": 75,
                "tp": 2,
                "recall": 0.02666666666666667,
                "recall_wilson_95": [
                  0.0073436890061313945,
                  0.09211488187759151
                ]
              },
              "non_violent_unethical_behavior": {
                "support": 608,
                "tp": 24,
                "recall": 0.039473684210526314,
                "recall_wilson_95": [
                  0.026667540134386394,
                  0.05806267563199362
                ]
              },
              "privacy_violation": {
                "support": 155,
                "tp": 2,
                "recall": 0.012903225806451613,
                "recall_wilson_95": [
                  0.0035456916701159073,
                  0.04582088352323989
                ]
              },
              "self_harm": {
                "support": 19,
                "tp": 1,
                "recall": 0.05263157894736842,
                "recall_wilson_95": [
                  0.009351994062300362,
                  0.24638731166898786
                ]
              },
              "sexually_explicit,adult_content": {
                "support": 103,
                "tp": 10,
                "recall": 0.0970873786407767,
                "recall_wilson_95": [
                  0.05359377103943983,
                  0.16955423801859704
                ]
              },
              "terrorism,organized_crime": {
                "support": 43,
                "tp": 0,
                "recall": 0.0,
                "recall_wilson_95": [
                  0.0,
                  0.08200980322579117
                ]
              },
              "violence,aiding_and_abetting,incitement": {
                "support": 724,
                "tp": 14,
                "recall": 0.019337016574585635,
                "recall_wilson_95": [
                  0.011553011612310182,
                  0.032194783233405466
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 3021,
            "summary": {
              "n": 3021,
              "correct": 1304,
              "accuracy": 0.43164515061238,
              "wilson_95": [
                0.41408071791152434,
                0.4493832005706635
              ],
              "macro_f1": 0.3264266527979867,
              "errors": 0,
              "probability_rows": 3021,
              "multiclass_brier": 0.8814790772591856,
              "nll": 1.307393055712035,
              "ece_10_equal_width": 0.4396135716650116,
              "p50_ms": 42.67820809036493,
              "p95_ms": 214.3910420127213,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 3021,
              "per_class": {
                "safe": {
                  "support": 1288,
                  "negative_support": 1733,
                  "tp": 1249,
                  "fp": 1678,
                  "fn": 39,
                  "tn": 55,
                  "precision": 0.42671677485480014,
                  "recall": 0.96972049689441,
                  "recall_wilson_95": [
                    0.9588757242210745,
                    0.9777717194246759
                  ],
                  "false_positive_rate": 0.9682631275245239,
                  "false_positive_wilson_95": [
                    0.9589184804554151,
                    0.9755364127977711
                  ]
                },
                "unsafe": {
                  "support": 1733,
                  "negative_support": 1288,
                  "tp": 55,
                  "fp": 39,
                  "fn": 1678,
                  "tn": 1249,
                  "precision": 0.5851063829787234,
                  "recall": 0.031736872475476054,
                  "recall_wilson_95": [
                    0.024463587202228878,
                    0.04108151954458489
                  ],
                  "false_positive_rate": 0.030279503105590064,
                  "false_positive_wilson_95": [
                    0.022228280575324118,
                    0.04112427577892556
                  ]
                }
              },
              "majority_baseline": 0.5736511089043363,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 55,
                "fp": 39,
                "fn": 1678,
                "tn": 1249,
                "precision": 0.5851063829787234,
                "recall": 0.031736872475476054,
                "recall_wilson_95": [
                  0.024463587202228878,
                  0.04108151954458489
                ],
                "false_positive_rate": 0.030279503105590064,
                "false_positive_wilson_95": [
                  0.022228280575324118,
                  0.04112427577892556
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 55,
                "fp": 39,
                "fn": 1678,
                "tn": 1249,
                "precision": 0.5851063829787234,
                "recall": 0.031736872475476054,
                "recall_wilson_95": [
                  0.024463587202228878,
                  0.04108151954458489
                ],
                "false_positive_rate": 0.030279503105590064,
                "false_positive_wilson_95": [
                  0.022228280575324118,
                  0.04112427577892556
                ],
                "f1": 0.060207991242474,
                "negative_errors": 0
              },
              "harmful_f1": 0.060207991242474,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.6312380584550155,
                "scored_rows": 3021,
                "excluded_rows": 0,
                "positive_support": 1733,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "animal_abuse": {
                  "support": 44,
                  "tp": 2,
                  "recall": 0.045454545454545456,
                  "recall_wilson_95": [
                    0.012555109949594967,
                    0.15134998437629657
                  ]
                },
                "child_abuse": {
                  "support": 27,
                  "tp": 0,
                  "recall": 0.0,
                  "recall_wilson_95": [
                    0.0,
                    0.12455502974186705
                  ]
                },
                "controversial_topics,politics": {
                  "support": 95,
                  "tp": 3,
                  "recall": 0.031578947368421054,
                  "recall_wilson_95": [
                    0.010797139474424379,
                    0.08877098647308188
                  ]
                },
                "discrimination,stereotype,injustice": {
                  "support": 297,
                  "tp": 7,
                  "recall": 0.02356902356902357,
                  "recall_wilson_95": [
                    0.0114627442740084,
                    0.04784244222067581
                  ]
                },
                "drug_abuse,weapons,banned_substance": {
                  "support": 151,
                  "tp": 2,
                  "recall": 0.013245033112582781,
                  "recall_wilson_95": [
                    0.0036398128620662405,
                    0.0470020428813268
                  ]
                },
                "financial_crime,property_crime,theft": {
                  "support": 262,
                  "tp": 5,
                  "recall": 0.019083969465648856,
                  "recall_wilson_95": [
                    0.008178399215518155,
                    0.04388819296350676
                  ]
                },
                "hate_speech,offensive_language": {
                  "support": 302,
                  "tp": 26,
                  "recall": 0.08609271523178808,
                  "recall_wilson_95": [
                    0.05942803805385782,
                    0.12315498726918161
                  ]
                },
                "misinformation_regarding_ethics,laws_and_safety": {
                  "support": 75,
                  "tp": 2,
                  "recall": 0.02666666666666667,
                  "recall_wilson_95": [
                    0.0073436890061313945,
                    0.09211488187759151
                  ]
                },
                "non_violent_unethical_behavior": {
                  "support": 608,
                  "tp": 24,
                  "recall": 0.039473684210526314,
                  "recall_wilson_95": [
                    0.026667540134386394,
                    0.05806267563199362
                  ]
                },
                "privacy_violation": {
                  "support": 155,
                  "tp": 2,
                  "recall": 0.012903225806451613,
                  "recall_wilson_95": [
                    0.0035456916701159073,
                    0.04582088352323989
                  ]
                },
                "self_harm": {
                  "support": 19,
                  "tp": 1,
                  "recall": 0.05263157894736842,
                  "recall_wilson_95": [
                    0.009351994062300362,
                    0.24638731166898786
                  ]
                },
                "sexually_explicit,adult_content": {
                  "support": 103,
                  "tp": 10,
                  "recall": 0.0970873786407767,
                  "recall_wilson_95": [
                    0.05359377103943983,
                    0.16955423801859704
                  ]
                },
                "terrorism,organized_crime": {
                  "support": 43,
                  "tp": 0,
                  "recall": 0.0,
                  "recall_wilson_95": [
                    0.0,
                    0.08200980322579117
                  ]
                },
                "violence,aiding_and_abetting,incitement": {
                  "support": 724,
                  "tp": 14,
                  "recall": 0.019337016574585635,
                  "recall_wilson_95": [
                    0.011553011612310182,
                    0.032194783233405466
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/beavertails_fine1000.json",
          "input_sha256": "151e230f199fa2564702da5d96c384113ad5846c824c9d27db6982e3b405e4ba",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 3021,
            "correct": 1630,
            "accuracy": 0.5395564382654751,
            "wilson_95": [
              0.5217436589830747,
              0.5572687465441822
            ],
            "macro_f1": 0.49360830074348594,
            "errors": 0,
            "probability_rows": 3021,
            "multiclass_brier": 0.7545980619938298,
            "nll": 1.3091696805074802,
            "ece_10_equal_width": 0.36151045726504866,
            "p50_ms": 43.63525006920099,
            "p95_ms": 62.15275009162724,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 3021,
            "per_class": {
              "safe": {
                "support": 1288,
                "negative_support": 1733,
                "tp": 1270,
                "fp": 1373,
                "fn": 18,
                "tn": 360,
                "precision": 0.48051456678017407,
                "recall": 0.9860248447204969,
                "recall_wilson_95": [
                  0.9780171974461451,
                  0.9911419756484661
                ],
                "false_positive_rate": 0.792267743796884,
                "false_positive_wilson_95": [
                  0.7725313575470591,
                  0.8107112837748618
                ]
              },
              "unsafe": {
                "support": 1733,
                "negative_support": 1288,
                "tp": 360,
                "fp": 18,
                "fn": 1373,
                "tn": 1270,
                "precision": 0.9523809523809523,
                "recall": 0.20773225620311597,
                "recall_wilson_95": [
                  0.18928871622513826,
                  0.2274686424529408
                ],
                "false_positive_rate": 0.013975155279503106,
                "false_positive_wilson_95": [
                  0.00885802435153394,
                  0.02198280255385497
                ]
              }
            },
            "majority_baseline": 0.5736511089043363,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 360,
              "fp": 18,
              "fn": 1373,
              "tn": 1270,
              "precision": 0.9523809523809523,
              "recall": 0.20773225620311597,
              "recall_wilson_95": [
                0.18928871622513826,
                0.2274686424529408
              ],
              "false_positive_rate": 0.013975155279503106,
              "false_positive_wilson_95": [
                0.00885802435153394,
                0.02198280255385497
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 360,
              "fp": 18,
              "fn": 1373,
              "tn": 1270,
              "precision": 0.9523809523809523,
              "recall": 0.20773225620311597,
              "recall_wilson_95": [
                0.18928871622513826,
                0.2274686424529408
              ],
              "false_positive_rate": 0.013975155279503106,
              "false_positive_wilson_95": [
                0.00885802435153394,
                0.02198280255385497
              ],
              "f1": 0.3410705826622454,
              "negative_errors": 0
            },
            "harmful_f1": 0.3410705826622454,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8385334733114209,
              "scored_rows": 3021,
              "excluded_rows": 0,
              "positive_support": 1733,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "animal_abuse": {
                "support": 44,
                "tp": 14,
                "recall": 0.3181818181818182,
                "recall_wilson_95": [
                  0.19999307710853426,
                  0.46556896062182235
                ]
              },
              "child_abuse": {
                "support": 27,
                "tp": 8,
                "recall": 0.2962962962962963,
                "recall_wilson_95": [
                  0.1585304046977976,
                  0.4848068296414814
                ]
              },
              "controversial_topics,politics": {
                "support": 95,
                "tp": 4,
                "recall": 0.042105263157894736,
                "recall_wilson_95": [
                  0.016493772862549924,
                  0.10330877722096742
                ]
              },
              "discrimination,stereotype,injustice": {
                "support": 297,
                "tp": 43,
                "recall": 0.1447811447811448,
                "recall_wilson_95": [
                  0.10929656357518637,
                  0.18933733872297043
                ]
              },
              "drug_abuse,weapons,banned_substance": {
                "support": 151,
                "tp": 29,
                "recall": 0.19205298013245034,
                "recall_wilson_95": [
                  0.13717928928278125,
                  0.2622063745548756
                ]
              },
              "financial_crime,property_crime,theft": {
                "support": 262,
                "tp": 99,
                "recall": 0.37786259541984735,
                "recall_wilson_95": [
                  0.32131712430582193,
                  0.43793788323170824
                ]
              },
              "hate_speech,offensive_language": {
                "support": 302,
                "tp": 76,
                "recall": 0.25165562913907286,
                "recall_wilson_95": [
                  0.20603941437581688,
                  0.30351040081800673
                ]
              },
              "misinformation_regarding_ethics,laws_and_safety": {
                "support": 75,
                "tp": 11,
                "recall": 0.14666666666666667,
                "recall_wilson_95": [
                  0.08391717009472589,
                  0.24384767859312362
                ]
              },
              "non_violent_unethical_behavior": {
                "support": 608,
                "tp": 111,
                "recall": 0.1825657894736842,
                "recall_wilson_95": [
                  0.15388389293310736,
                  0.21523372007729033
                ]
              },
              "privacy_violation": {
                "support": 155,
                "tp": 21,
                "recall": 0.13548387096774195,
                "recall_wilson_95": [
                  0.09035162045827663,
                  0.1982472073354268
                ]
              },
              "self_harm": {
                "support": 19,
                "tp": 6,
                "recall": 0.3157894736842105,
                "recall_wilson_95": [
                  0.1536437908779228,
                  0.5398959232467253
                ]
              },
              "sexually_explicit,adult_content": {
                "support": 103,
                "tp": 27,
                "recall": 0.2621359223300971,
                "recall_wilson_95": [
                  0.1868578650291558,
                  0.3545186704388419
                ]
              },
              "terrorism,organized_crime": {
                "support": 43,
                "tp": 12,
                "recall": 0.27906976744186046,
                "recall_wilson_95": [
                  0.16746613786237488,
                  0.42691028681878873
                ]
              },
              "violence,aiding_and_abetting,incitement": {
                "support": 724,
                "tp": 212,
                "recall": 0.292817679558011,
                "recall_wilson_95": [
                  0.26083375391288094,
                  0.32698857145165167
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 3021,
            "summary": {
              "n": 3021,
              "correct": 1630,
              "accuracy": 0.5395564382654751,
              "wilson_95": [
                0.5217436589830747,
                0.5572687465441822
              ],
              "macro_f1": 0.49360830074348594,
              "errors": 0,
              "probability_rows": 3021,
              "multiclass_brier": 0.7545980619938298,
              "nll": 1.3091696805074802,
              "ece_10_equal_width": 0.36151045726504866,
              "p50_ms": 43.63525006920099,
              "p95_ms": 62.15275009162724,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 3021,
              "per_class": {
                "safe": {
                  "support": 1288,
                  "negative_support": 1733,
                  "tp": 1270,
                  "fp": 1373,
                  "fn": 18,
                  "tn": 360,
                  "precision": 0.48051456678017407,
                  "recall": 0.9860248447204969,
                  "recall_wilson_95": [
                    0.9780171974461451,
                    0.9911419756484661
                  ],
                  "false_positive_rate": 0.792267743796884,
                  "false_positive_wilson_95": [
                    0.7725313575470591,
                    0.8107112837748618
                  ]
                },
                "unsafe": {
                  "support": 1733,
                  "negative_support": 1288,
                  "tp": 360,
                  "fp": 18,
                  "fn": 1373,
                  "tn": 1270,
                  "precision": 0.9523809523809523,
                  "recall": 0.20773225620311597,
                  "recall_wilson_95": [
                    0.18928871622513826,
                    0.2274686424529408
                  ],
                  "false_positive_rate": 0.013975155279503106,
                  "false_positive_wilson_95": [
                    0.00885802435153394,
                    0.02198280255385497
                  ]
                }
              },
              "majority_baseline": 0.5736511089043363,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 360,
                "fp": 18,
                "fn": 1373,
                "tn": 1270,
                "precision": 0.9523809523809523,
                "recall": 0.20773225620311597,
                "recall_wilson_95": [
                  0.18928871622513826,
                  0.2274686424529408
                ],
                "false_positive_rate": 0.013975155279503106,
                "false_positive_wilson_95": [
                  0.00885802435153394,
                  0.02198280255385497
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 360,
                "fp": 18,
                "fn": 1373,
                "tn": 1270,
                "precision": 0.9523809523809523,
                "recall": 0.20773225620311597,
                "recall_wilson_95": [
                  0.18928871622513826,
                  0.2274686424529408
                ],
                "false_positive_rate": 0.013975155279503106,
                "false_positive_wilson_95": [
                  0.00885802435153394,
                  0.02198280255385497
                ],
                "f1": 0.3410705826622454,
                "negative_errors": 0
              },
              "harmful_f1": 0.3410705826622454,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8385334733114209,
                "scored_rows": 3021,
                "excluded_rows": 0,
                "positive_support": 1733,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "animal_abuse": {
                  "support": 44,
                  "tp": 14,
                  "recall": 0.3181818181818182,
                  "recall_wilson_95": [
                    0.19999307710853426,
                    0.46556896062182235
                  ]
                },
                "child_abuse": {
                  "support": 27,
                  "tp": 8,
                  "recall": 0.2962962962962963,
                  "recall_wilson_95": [
                    0.1585304046977976,
                    0.4848068296414814
                  ]
                },
                "controversial_topics,politics": {
                  "support": 95,
                  "tp": 4,
                  "recall": 0.042105263157894736,
                  "recall_wilson_95": [
                    0.016493772862549924,
                    0.10330877722096742
                  ]
                },
                "discrimination,stereotype,injustice": {
                  "support": 297,
                  "tp": 43,
                  "recall": 0.1447811447811448,
                  "recall_wilson_95": [
                    0.10929656357518637,
                    0.18933733872297043
                  ]
                },
                "drug_abuse,weapons,banned_substance": {
                  "support": 151,
                  "tp": 29,
                  "recall": 0.19205298013245034,
                  "recall_wilson_95": [
                    0.13717928928278125,
                    0.2622063745548756
                  ]
                },
                "financial_crime,property_crime,theft": {
                  "support": 262,
                  "tp": 99,
                  "recall": 0.37786259541984735,
                  "recall_wilson_95": [
                    0.32131712430582193,
                    0.43793788323170824
                  ]
                },
                "hate_speech,offensive_language": {
                  "support": 302,
                  "tp": 76,
                  "recall": 0.25165562913907286,
                  "recall_wilson_95": [
                    0.20603941437581688,
                    0.30351040081800673
                  ]
                },
                "misinformation_regarding_ethics,laws_and_safety": {
                  "support": 75,
                  "tp": 11,
                  "recall": 0.14666666666666667,
                  "recall_wilson_95": [
                    0.08391717009472589,
                    0.24384767859312362
                  ]
                },
                "non_violent_unethical_behavior": {
                  "support": 608,
                  "tp": 111,
                  "recall": 0.1825657894736842,
                  "recall_wilson_95": [
                    0.15388389293310736,
                    0.21523372007729033
                  ]
                },
                "privacy_violation": {
                  "support": 155,
                  "tp": 21,
                  "recall": 0.13548387096774195,
                  "recall_wilson_95": [
                    0.09035162045827663,
                    0.1982472073354268
                  ]
                },
                "self_harm": {
                  "support": 19,
                  "tp": 6,
                  "recall": 0.3157894736842105,
                  "recall_wilson_95": [
                    0.1536437908779228,
                    0.5398959232467253
                  ]
                },
                "sexually_explicit,adult_content": {
                  "support": 103,
                  "tp": 27,
                  "recall": 0.2621359223300971,
                  "recall_wilson_95": [
                    0.1868578650291558,
                    0.3545186704388419
                  ]
                },
                "terrorism,organized_crime": {
                  "support": 43,
                  "tp": 12,
                  "recall": 0.27906976744186046,
                  "recall_wilson_95": [
                    0.16746613786237488,
                    0.42691028681878873
                  ]
                },
                "violence,aiding_and_abetting,incitement": {
                  "support": 724,
                  "tp": 212,
                  "recall": 0.292817679558011,
                  "recall_wilson_95": [
                    0.26083375391288094,
                    0.32698857145165167
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/beavertails_fine5000.json",
          "input_sha256": "39c0094e50529953de98a1d0e3b748122ea316cee9ce7c567e40239c93774258",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 3021,
            "correct": 1986,
            "accuracy": 0.6573982125124131,
            "wilson_95": [
              0.6402847163685985,
              0.6741119265452982
            ],
            "macro_f1": 0.6488257577757095,
            "errors": 0,
            "probability_rows": 3021,
            "multiclass_brier": 0.5547271160794709,
            "nll": 0.8917164439913337,
            "ece_10_equal_width": 0.2385971309125295,
            "p50_ms": 43.1212920229882,
            "p95_ms": 61.45979184657335,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 3021,
            "per_class": {
              "safe": {
                "support": 1288,
                "negative_support": 1733,
                "tp": 1229,
                "fp": 976,
                "fn": 59,
                "tn": 757,
                "precision": 0.5573696145124717,
                "recall": 0.9541925465838509,
                "recall_wilson_95": [
                  0.9413615750559576,
                  0.9643223167337347
                ],
                "false_positive_rate": 0.5631852279284478,
                "false_positive_wilson_95": [
                  0.5397189510526176,
                  0.5863720049686073
                ]
              },
              "unsafe": {
                "support": 1733,
                "negative_support": 1288,
                "tp": 757,
                "fp": 59,
                "fn": 976,
                "tn": 1229,
                "precision": 0.9276960784313726,
                "recall": 0.4368147720715522,
                "recall_wilson_95": [
                  0.41362799503139275,
                  0.4602810489473825
                ],
                "false_positive_rate": 0.04580745341614907,
                "false_positive_wilson_95": [
                  0.03567768326626524,
                  0.058638424944042304
                ]
              }
            },
            "majority_baseline": 0.5736511089043363,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 757,
              "fp": 59,
              "fn": 976,
              "tn": 1229,
              "precision": 0.9276960784313726,
              "recall": 0.4368147720715522,
              "recall_wilson_95": [
                0.41362799503139275,
                0.4602810489473825
              ],
              "false_positive_rate": 0.04580745341614907,
              "false_positive_wilson_95": [
                0.03567768326626524,
                0.058638424944042304
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 1733,
              "negative_support": 1288,
              "tp": 757,
              "fp": 59,
              "fn": 976,
              "tn": 1229,
              "precision": 0.9276960784313726,
              "recall": 0.4368147720715522,
              "recall_wilson_95": [
                0.41362799503139275,
                0.4602810489473825
              ],
              "false_positive_rate": 0.04580745341614907,
              "false_positive_wilson_95": [
                0.03567768326626524,
                0.058638424944042304
              ],
              "f1": 0.5939584150647312,
              "negative_errors": 0
            },
            "harmful_f1": 0.5939584150647312,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8797517139679273,
              "scored_rows": 3021,
              "excluded_rows": 0,
              "positive_support": 1733,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "animal_abuse": {
                "support": 44,
                "tp": 26,
                "recall": 0.5909090909090909,
                "recall_wilson_95": [
                  0.444097537410319,
                  0.7231214437245027
                ]
              },
              "child_abuse": {
                "support": 27,
                "tp": 17,
                "recall": 0.6296296296296297,
                "recall_wilson_95": [
                  0.4422923441210239,
                  0.784674870390344
                ]
              },
              "controversial_topics,politics": {
                "support": 95,
                "tp": 11,
                "recall": 0.11578947368421053,
                "recall_wilson_95": [
                  0.06589857581116428,
                  0.19554494322443078
                ]
              },
              "discrimination,stereotype,injustice": {
                "support": 297,
                "tp": 85,
                "recall": 0.28619528619528617,
                "recall_wilson_95": [
                  0.2377784169900016,
                  0.3400723204122065
                ]
              },
              "drug_abuse,weapons,banned_substance": {
                "support": 151,
                "tp": 94,
                "recall": 0.6225165562913907,
                "recall_wilson_95": [
                  0.5430630840583557,
                  0.6958910066728992
                ]
              },
              "financial_crime,property_crime,theft": {
                "support": 262,
                "tp": 180,
                "recall": 0.6870229007633588,
                "recall_wilson_95": [
                  0.628513413925545,
                  0.740127355782612
                ]
              },
              "hate_speech,offensive_language": {
                "support": 302,
                "tp": 110,
                "recall": 0.36423841059602646,
                "recall_wilson_95": [
                  0.31198548463531844,
                  0.4199017476706383
                ]
              },
              "misinformation_regarding_ethics,laws_and_safety": {
                "support": 75,
                "tp": 27,
                "recall": 0.36,
                "recall_wilson_95": [
                  0.26064928908616347,
                  0.47299338680902214
                ]
              },
              "non_violent_unethical_behavior": {
                "support": 608,
                "tp": 228,
                "recall": 0.375,
                "recall_wilson_95": [
                  0.3374162855267025,
                  0.41415334446702917
                ]
              },
              "privacy_violation": {
                "support": 155,
                "tp": 61,
                "recall": 0.3935483870967742,
                "recall_wilson_95": [
                  0.32010552662008906,
                  0.47214014875329346
                ]
              },
              "self_harm": {
                "support": 19,
                "tp": 11,
                "recall": 0.5789473684210527,
                "recall_wilson_95": [
                  0.3627590353425504,
                  0.7685810871754576
                ]
              },
              "sexually_explicit,adult_content": {
                "support": 103,
                "tp": 48,
                "recall": 0.46601941747572817,
                "recall_wilson_95": [
                  0.3726437920092655,
                  0.5618385702004485
                ]
              },
              "terrorism,organized_crime": {
                "support": 43,
                "tp": 31,
                "recall": 0.7209302325581395,
                "recall_wilson_95": [
                  0.5730897131812113,
                  0.8325338621376251
                ]
              },
              "violence,aiding_and_abetting,incitement": {
                "support": 724,
                "tp": 458,
                "recall": 0.6325966850828729,
                "recall_wilson_95": [
                  0.5968659500211134,
                  0.6669277617455857
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 3021,
            "summary": {
              "n": 3021,
              "correct": 1986,
              "accuracy": 0.6573982125124131,
              "wilson_95": [
                0.6402847163685985,
                0.6741119265452982
              ],
              "macro_f1": 0.6488257577757095,
              "errors": 0,
              "probability_rows": 3021,
              "multiclass_brier": 0.5547271160794709,
              "nll": 0.8917164439913337,
              "ece_10_equal_width": 0.2385971309125295,
              "p50_ms": 43.1212920229882,
              "p95_ms": 61.45979184657335,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 3021,
              "per_class": {
                "safe": {
                  "support": 1288,
                  "negative_support": 1733,
                  "tp": 1229,
                  "fp": 976,
                  "fn": 59,
                  "tn": 757,
                  "precision": 0.5573696145124717,
                  "recall": 0.9541925465838509,
                  "recall_wilson_95": [
                    0.9413615750559576,
                    0.9643223167337347
                  ],
                  "false_positive_rate": 0.5631852279284478,
                  "false_positive_wilson_95": [
                    0.5397189510526176,
                    0.5863720049686073
                  ]
                },
                "unsafe": {
                  "support": 1733,
                  "negative_support": 1288,
                  "tp": 757,
                  "fp": 59,
                  "fn": 976,
                  "tn": 1229,
                  "precision": 0.9276960784313726,
                  "recall": 0.4368147720715522,
                  "recall_wilson_95": [
                    0.41362799503139275,
                    0.4602810489473825
                  ],
                  "false_positive_rate": 0.04580745341614907,
                  "false_positive_wilson_95": [
                    0.03567768326626524,
                    0.058638424944042304
                  ]
                }
              },
              "majority_baseline": 0.5736511089043363,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 757,
                "fp": 59,
                "fn": 976,
                "tn": 1229,
                "precision": 0.9276960784313726,
                "recall": 0.4368147720715522,
                "recall_wilson_95": [
                  0.41362799503139275,
                  0.4602810489473825
                ],
                "false_positive_rate": 0.04580745341614907,
                "false_positive_wilson_95": [
                  0.03567768326626524,
                  0.058638424944042304
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 1733,
                "negative_support": 1288,
                "tp": 757,
                "fp": 59,
                "fn": 976,
                "tn": 1229,
                "precision": 0.9276960784313726,
                "recall": 0.4368147720715522,
                "recall_wilson_95": [
                  0.41362799503139275,
                  0.4602810489473825
                ],
                "false_positive_rate": 0.04580745341614907,
                "false_positive_wilson_95": [
                  0.03567768326626524,
                  0.058638424944042304
                ],
                "f1": 0.5939584150647312,
                "negative_errors": 0
              },
              "harmful_f1": 0.5939584150647312,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8797517139679273,
                "scored_rows": 3021,
                "excluded_rows": 0,
                "positive_support": 1733,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "animal_abuse": {
                  "support": 44,
                  "tp": 26,
                  "recall": 0.5909090909090909,
                  "recall_wilson_95": [
                    0.444097537410319,
                    0.7231214437245027
                  ]
                },
                "child_abuse": {
                  "support": 27,
                  "tp": 17,
                  "recall": 0.6296296296296297,
                  "recall_wilson_95": [
                    0.4422923441210239,
                    0.784674870390344
                  ]
                },
                "controversial_topics,politics": {
                  "support": 95,
                  "tp": 11,
                  "recall": 0.11578947368421053,
                  "recall_wilson_95": [
                    0.06589857581116428,
                    0.19554494322443078
                  ]
                },
                "discrimination,stereotype,injustice": {
                  "support": 297,
                  "tp": 85,
                  "recall": 0.28619528619528617,
                  "recall_wilson_95": [
                    0.2377784169900016,
                    0.3400723204122065
                  ]
                },
                "drug_abuse,weapons,banned_substance": {
                  "support": 151,
                  "tp": 94,
                  "recall": 0.6225165562913907,
                  "recall_wilson_95": [
                    0.5430630840583557,
                    0.6958910066728992
                  ]
                },
                "financial_crime,property_crime,theft": {
                  "support": 262,
                  "tp": 180,
                  "recall": 0.6870229007633588,
                  "recall_wilson_95": [
                    0.628513413925545,
                    0.740127355782612
                  ]
                },
                "hate_speech,offensive_language": {
                  "support": 302,
                  "tp": 110,
                  "recall": 0.36423841059602646,
                  "recall_wilson_95": [
                    0.31198548463531844,
                    0.4199017476706383
                  ]
                },
                "misinformation_regarding_ethics,laws_and_safety": {
                  "support": 75,
                  "tp": 27,
                  "recall": 0.36,
                  "recall_wilson_95": [
                    0.26064928908616347,
                    0.47299338680902214
                  ]
                },
                "non_violent_unethical_behavior": {
                  "support": 608,
                  "tp": 228,
                  "recall": 0.375,
                  "recall_wilson_95": [
                    0.3374162855267025,
                    0.41415334446702917
                  ]
                },
                "privacy_violation": {
                  "support": 155,
                  "tp": 61,
                  "recall": 0.3935483870967742,
                  "recall_wilson_95": [
                    0.32010552662008906,
                    0.47214014875329346
                  ]
                },
                "self_harm": {
                  "support": 19,
                  "tp": 11,
                  "recall": 0.5789473684210527,
                  "recall_wilson_95": [
                    0.3627590353425504,
                    0.7685810871754576
                  ]
                },
                "sexually_explicit,adult_content": {
                  "support": 103,
                  "tp": 48,
                  "recall": 0.46601941747572817,
                  "recall_wilson_95": [
                    0.3726437920092655,
                    0.5618385702004485
                  ]
                },
                "terrorism,organized_crime": {
                  "support": 43,
                  "tp": 31,
                  "recall": 0.7209302325581395,
                  "recall_wilson_95": [
                    0.5730897131812113,
                    0.8325338621376251
                  ]
                },
                "violence,aiding_and_abetting,incitement": {
                  "support": 724,
                  "tp": 458,
                  "recall": 0.6325966850828729,
                  "recall_wilson_95": [
                    0.5968659500211134,
                    0.6669277617455857
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.10791128765309504,
          "harmful_f1_delta_fine_minus_reference": 0.2808625914197714,
          "accuracy_interval": [
            0.094602925650977,
            0.12110204086102047
          ],
          "harmful_f1_interval": [
            0.2508489292600077,
            0.31015581904380446
          ],
          "confidence": 0.95,
          "n_rows": 3021,
          "n_text_groups": 2523,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.22575306190003314,
          "harmful_f1_delta_fine_minus_reference": 0.5337504238222572,
          "accuracy_interval": [
            0.20712382234126128,
            0.2448038663627959
          ],
          "harmful_f1_interval": [
            0.5045555115723762,
            0.561483665464481
          ],
          "confidence": 0.95,
          "n_rows": 3021,
          "n_text_groups": 2523,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.11784177424693809,
          "harmful_f1_delta_fine_minus_reference": 0.25288783240248586,
          "accuracy_interval": [
            0.10313972487860296,
            0.13273635175408544
          ],
          "harmful_f1_interval": [
            0.22714463208850175,
            0.27887829816186366
          ],
          "confidence": 0.95,
          "n_rows": 3021,
          "n_text_groups": 2523,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "wildguard_prompt",
      "n": 1699,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_prompt_base.json",
          "input_sha256": "8a050aa2d2581a804f6cb719f979afe4d3ba3099e04e35fded476171e91dd39b",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1699,
            "correct": 1108,
            "accuracy": 0.6521483225426722,
            "wilson_95": [
              0.6291804739298713,
              0.6744297049989699
            ],
            "macro_f1": 0.6120862203180625,
            "errors": 0,
            "probability_rows": 1699,
            "multiclass_brier": 0.4416541520578879,
            "nll": 0.6362969648025371,
            "ece_10_equal_width": 0.06434627194422371,
            "p50_ms": 43.70062495581806,
            "p95_ms": 206.89237513579428,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1699,
            "per_class": {
              "safe": {
                "support": 945,
                "negative_support": 754,
                "tp": 827,
                "fp": 473,
                "fn": 118,
                "tn": 281,
                "precision": 0.6361538461538462,
                "recall": 0.8751322751322751,
                "recall_wilson_95": [
                  0.8525251816766027,
                  0.8947018639050787
                ],
                "false_positive_rate": 0.6273209549071618,
                "false_positive_wilson_95": [
                  0.5922447327976477,
                  0.6611064104596485
                ]
              },
              "unsafe": {
                "support": 754,
                "negative_support": 945,
                "tp": 281,
                "fp": 118,
                "fn": 473,
                "tn": 827,
                "precision": 0.7042606516290727,
                "recall": 0.3726790450928382,
                "recall_wilson_95": [
                  0.33889358954035165,
                  0.40775526720235233
                ],
                "false_positive_rate": 0.12486772486772486,
                "false_positive_wilson_95": [
                  0.10529813609492127,
                  0.1474748183233972
                ]
              }
            },
            "majority_baseline": 0.5562095350206003,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 281,
              "fp": 118,
              "fn": 473,
              "tn": 827,
              "precision": 0.7042606516290727,
              "recall": 0.3726790450928382,
              "recall_wilson_95": [
                0.33889358954035165,
                0.40775526720235233
              ],
              "false_positive_rate": 0.12486772486772486,
              "false_positive_wilson_95": [
                0.10529813609492127,
                0.1474748183233972
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 281,
              "fp": 118,
              "fn": 473,
              "tn": 827,
              "precision": 0.7042606516290727,
              "recall": 0.3726790450928382,
              "recall_wilson_95": [
                0.33889358954035165,
                0.40775526720235233
              ],
              "false_positive_rate": 0.12486772486772486,
              "false_positive_wilson_95": [
                0.10529813609492127,
                0.1474748183233972
              ],
              "f1": 0.48742411101474414,
              "negative_errors": 0
            },
            "harmful_f1": 0.48742411101474414,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.6761381998368621,
              "scored_rows": 1699,
              "excluded_rows": 0,
              "positive_support": 754,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1699,
            "summary": {
              "n": 1699,
              "correct": 1108,
              "accuracy": 0.6521483225426722,
              "wilson_95": [
                0.6291804739298713,
                0.6744297049989699
              ],
              "macro_f1": 0.6120862203180625,
              "errors": 0,
              "probability_rows": 1699,
              "multiclass_brier": 0.4416541520578879,
              "nll": 0.6362969648025371,
              "ece_10_equal_width": 0.06434627194422371,
              "p50_ms": 43.70062495581806,
              "p95_ms": 206.89237513579428,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1699,
              "per_class": {
                "safe": {
                  "support": 945,
                  "negative_support": 754,
                  "tp": 827,
                  "fp": 473,
                  "fn": 118,
                  "tn": 281,
                  "precision": 0.6361538461538462,
                  "recall": 0.8751322751322751,
                  "recall_wilson_95": [
                    0.8525251816766027,
                    0.8947018639050787
                  ],
                  "false_positive_rate": 0.6273209549071618,
                  "false_positive_wilson_95": [
                    0.5922447327976477,
                    0.6611064104596485
                  ]
                },
                "unsafe": {
                  "support": 754,
                  "negative_support": 945,
                  "tp": 281,
                  "fp": 118,
                  "fn": 473,
                  "tn": 827,
                  "precision": 0.7042606516290727,
                  "recall": 0.3726790450928382,
                  "recall_wilson_95": [
                    0.33889358954035165,
                    0.40775526720235233
                  ],
                  "false_positive_rate": 0.12486772486772486,
                  "false_positive_wilson_95": [
                    0.10529813609492127,
                    0.1474748183233972
                  ]
                }
              },
              "majority_baseline": 0.5562095350206003,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 281,
                "fp": 118,
                "fn": 473,
                "tn": 827,
                "precision": 0.7042606516290727,
                "recall": 0.3726790450928382,
                "recall_wilson_95": [
                  0.33889358954035165,
                  0.40775526720235233
                ],
                "false_positive_rate": 0.12486772486772486,
                "false_positive_wilson_95": [
                  0.10529813609492127,
                  0.1474748183233972
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 281,
                "fp": 118,
                "fn": 473,
                "tn": 827,
                "precision": 0.7042606516290727,
                "recall": 0.3726790450928382,
                "recall_wilson_95": [
                  0.33889358954035165,
                  0.40775526720235233
                ],
                "false_positive_rate": 0.12486772486772486,
                "false_positive_wilson_95": [
                  0.10529813609492127,
                  0.1474748183233972
                ],
                "f1": 0.48742411101474414,
                "negative_errors": 0
              },
              "harmful_f1": 0.48742411101474414,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.6761381998368621,
                "scored_rows": 1699,
                "excluded_rows": 0,
                "positive_support": 754,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_prompt_fine1000.json",
          "input_sha256": "b786c1e4771b5c23d0498dd7fe9e797dfedc86ec1f531f4629df2a1e3e63ebfe",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1699,
            "correct": 1162,
            "accuracy": 0.6839317245438493,
            "wilson_95": [
              0.6614299063110399,
              0.705603675566186
            ],
            "macro_f1": 0.6676545638335656,
            "errors": 0,
            "probability_rows": 1699,
            "multiclass_brier": 0.43650273733961154,
            "nll": 0.653326888238941,
            "ece_10_equal_width": 0.14074096527369037,
            "p50_ms": 43.18533302284777,
            "p95_ms": 78.98666709661484,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1699,
            "per_class": {
              "safe": {
                "support": 945,
                "negative_support": 754,
                "tp": 769,
                "fp": 361,
                "fn": 176,
                "tn": 393,
                "precision": 0.6805309734513274,
                "recall": 0.8137566137566138,
                "recall_wilson_95": [
                  0.7876830578422377,
                  0.8372896333142322
                ],
                "false_positive_rate": 0.47877984084880637,
                "false_positive_wilson_95": [
                  0.44332104413779005,
                  0.5144537653193274
                ]
              },
              "unsafe": {
                "support": 754,
                "negative_support": 945,
                "tp": 393,
                "fp": 176,
                "fn": 361,
                "tn": 769,
                "precision": 0.6906854130052724,
                "recall": 0.5212201591511937,
                "recall_wilson_95": [
                  0.4855462346806727,
                  0.5566789558622101
                ],
                "false_positive_rate": 0.18624338624338624,
                "false_positive_wilson_95": [
                  0.16271036668576774,
                  0.21231694215776234
                ]
              }
            },
            "majority_baseline": 0.5562095350206003,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 393,
              "fp": 176,
              "fn": 361,
              "tn": 769,
              "precision": 0.6906854130052724,
              "recall": 0.5212201591511937,
              "recall_wilson_95": [
                0.4855462346806727,
                0.5566789558622101
              ],
              "false_positive_rate": 0.18624338624338624,
              "false_positive_wilson_95": [
                0.16271036668576774,
                0.21231694215776234
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 393,
              "fp": 176,
              "fn": 361,
              "tn": 769,
              "precision": 0.6906854130052724,
              "recall": 0.5212201591511937,
              "recall_wilson_95": [
                0.4855462346806727,
                0.5566789558622101
              ],
              "false_positive_rate": 0.18624338624338624,
              "false_positive_wilson_95": [
                0.16271036668576774,
                0.21231694215776234
              ],
              "f1": 0.5941043083900227,
              "negative_errors": 0
            },
            "harmful_f1": 0.5941043083900227,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.73639628752554,
              "scored_rows": 1699,
              "excluded_rows": 0,
              "positive_support": 754,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1699,
            "summary": {
              "n": 1699,
              "correct": 1162,
              "accuracy": 0.6839317245438493,
              "wilson_95": [
                0.6614299063110399,
                0.705603675566186
              ],
              "macro_f1": 0.6676545638335656,
              "errors": 0,
              "probability_rows": 1699,
              "multiclass_brier": 0.43650273733961154,
              "nll": 0.653326888238941,
              "ece_10_equal_width": 0.14074096527369037,
              "p50_ms": 43.18533302284777,
              "p95_ms": 78.98666709661484,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1699,
              "per_class": {
                "safe": {
                  "support": 945,
                  "negative_support": 754,
                  "tp": 769,
                  "fp": 361,
                  "fn": 176,
                  "tn": 393,
                  "precision": 0.6805309734513274,
                  "recall": 0.8137566137566138,
                  "recall_wilson_95": [
                    0.7876830578422377,
                    0.8372896333142322
                  ],
                  "false_positive_rate": 0.47877984084880637,
                  "false_positive_wilson_95": [
                    0.44332104413779005,
                    0.5144537653193274
                  ]
                },
                "unsafe": {
                  "support": 754,
                  "negative_support": 945,
                  "tp": 393,
                  "fp": 176,
                  "fn": 361,
                  "tn": 769,
                  "precision": 0.6906854130052724,
                  "recall": 0.5212201591511937,
                  "recall_wilson_95": [
                    0.4855462346806727,
                    0.5566789558622101
                  ],
                  "false_positive_rate": 0.18624338624338624,
                  "false_positive_wilson_95": [
                    0.16271036668576774,
                    0.21231694215776234
                  ]
                }
              },
              "majority_baseline": 0.5562095350206003,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 393,
                "fp": 176,
                "fn": 361,
                "tn": 769,
                "precision": 0.6906854130052724,
                "recall": 0.5212201591511937,
                "recall_wilson_95": [
                  0.4855462346806727,
                  0.5566789558622101
                ],
                "false_positive_rate": 0.18624338624338624,
                "false_positive_wilson_95": [
                  0.16271036668576774,
                  0.21231694215776234
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 393,
                "fp": 176,
                "fn": 361,
                "tn": 769,
                "precision": 0.6906854130052724,
                "recall": 0.5212201591511937,
                "recall_wilson_95": [
                  0.4855462346806727,
                  0.5566789558622101
                ],
                "false_positive_rate": 0.18624338624338624,
                "false_positive_wilson_95": [
                  0.16271036668576774,
                  0.21231694215776234
                ],
                "f1": 0.5941043083900227,
                "negative_errors": 0
              },
              "harmful_f1": 0.5941043083900227,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.73639628752554,
                "scored_rows": 1699,
                "excluded_rows": 0,
                "positive_support": 754,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_prompt_fine5000.json",
          "input_sha256": "06d12f551915aa8f3e32fedd9b3971aecb15de258bc4e7aa46fb4f65ec2d36bb",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1699,
            "correct": 1251,
            "accuracy": 0.7363154796939376,
            "wilson_95": [
              0.7148472208754468,
              0.756717525120413
            ],
            "macro_f1": 0.7215691438772076,
            "errors": 0,
            "probability_rows": 1699,
            "multiclass_brier": 0.404205081530312,
            "nll": 0.6272847827258653,
            "ece_10_equal_width": 0.1373103001765745,
            "p50_ms": 42.81333298422396,
            "p95_ms": 78.25320796109736,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1699,
            "per_class": {
              "safe": {
                "support": 945,
                "negative_support": 754,
                "tp": 821,
                "fp": 324,
                "fn": 124,
                "tn": 430,
                "precision": 0.7170305676855895,
                "recall": 0.8687830687830688,
                "recall_wilson_95": [
                  0.845754858642737,
                  0.8888251847570261
                ],
                "false_positive_rate": 0.4297082228116711,
                "false_positive_wilson_95": [
                  0.3948179882559499,
                  0.46531106807075134
                ]
              },
              "unsafe": {
                "support": 754,
                "negative_support": 945,
                "tp": 430,
                "fp": 124,
                "fn": 324,
                "tn": 821,
                "precision": 0.776173285198556,
                "recall": 0.5702917771883289,
                "recall_wilson_95": [
                  0.5346889319292488,
                  0.6051820117440502
                ],
                "false_positive_rate": 0.1312169312169312,
                "false_positive_wilson_95": [
                  0.11117481524297385,
                  0.15424514135726303
                ]
              }
            },
            "majority_baseline": 0.5562095350206003,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 430,
              "fp": 124,
              "fn": 324,
              "tn": 821,
              "precision": 0.776173285198556,
              "recall": 0.5702917771883289,
              "recall_wilson_95": [
                0.5346889319292488,
                0.6051820117440502
              ],
              "false_positive_rate": 0.1312169312169312,
              "false_positive_wilson_95": [
                0.11117481524297385,
                0.15424514135726303
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 754,
              "negative_support": 945,
              "tp": 430,
              "fp": 124,
              "fn": 324,
              "tn": 821,
              "precision": 0.776173285198556,
              "recall": 0.5702917771883289,
              "recall_wilson_95": [
                0.5346889319292488,
                0.6051820117440502
              ],
              "false_positive_rate": 0.1312169312169312,
              "false_positive_wilson_95": [
                0.11117481524297385,
                0.15424514135726303
              ],
              "f1": 0.6574923547400612,
              "negative_errors": 0
            },
            "harmful_f1": 0.6574923547400612,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.7890861290846592,
              "scored_rows": 1699,
              "excluded_rows": 0,
              "positive_support": 754,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 1699,
            "summary": {
              "n": 1699,
              "correct": 1251,
              "accuracy": 0.7363154796939376,
              "wilson_95": [
                0.7148472208754468,
                0.756717525120413
              ],
              "macro_f1": 0.7215691438772076,
              "errors": 0,
              "probability_rows": 1699,
              "multiclass_brier": 0.404205081530312,
              "nll": 0.6272847827258653,
              "ece_10_equal_width": 0.1373103001765745,
              "p50_ms": 42.81333298422396,
              "p95_ms": 78.25320796109736,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1699,
              "per_class": {
                "safe": {
                  "support": 945,
                  "negative_support": 754,
                  "tp": 821,
                  "fp": 324,
                  "fn": 124,
                  "tn": 430,
                  "precision": 0.7170305676855895,
                  "recall": 0.8687830687830688,
                  "recall_wilson_95": [
                    0.845754858642737,
                    0.8888251847570261
                  ],
                  "false_positive_rate": 0.4297082228116711,
                  "false_positive_wilson_95": [
                    0.3948179882559499,
                    0.46531106807075134
                  ]
                },
                "unsafe": {
                  "support": 754,
                  "negative_support": 945,
                  "tp": 430,
                  "fp": 124,
                  "fn": 324,
                  "tn": 821,
                  "precision": 0.776173285198556,
                  "recall": 0.5702917771883289,
                  "recall_wilson_95": [
                    0.5346889319292488,
                    0.6051820117440502
                  ],
                  "false_positive_rate": 0.1312169312169312,
                  "false_positive_wilson_95": [
                    0.11117481524297385,
                    0.15424514135726303
                  ]
                }
              },
              "majority_baseline": 0.5562095350206003,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 430,
                "fp": 124,
                "fn": 324,
                "tn": 821,
                "precision": 0.776173285198556,
                "recall": 0.5702917771883289,
                "recall_wilson_95": [
                  0.5346889319292488,
                  0.6051820117440502
                ],
                "false_positive_rate": 0.1312169312169312,
                "false_positive_wilson_95": [
                  0.11117481524297385,
                  0.15424514135726303
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 754,
                "negative_support": 945,
                "tp": 430,
                "fp": 124,
                "fn": 324,
                "tn": 821,
                "precision": 0.776173285198556,
                "recall": 0.5702917771883289,
                "recall_wilson_95": [
                  0.5346889319292488,
                  0.6051820117440502
                ],
                "false_positive_rate": 0.1312169312169312,
                "false_positive_wilson_95": [
                  0.11117481524297385,
                  0.15424514135726303
                ],
                "f1": 0.6574923547400612,
                "negative_errors": 0
              },
              "harmful_f1": 0.6574923547400612,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.7890861290846592,
                "scored_rows": 1699,
                "excluded_rows": 0,
                "positive_support": 754,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.03178340200117713,
          "harmful_f1_delta_fine_minus_reference": 0.10668019737527856,
          "accuracy_interval": [
            0.009417304296645113,
            0.053560918187168904
          ],
          "harmful_f1_interval": [
            0.07375427713462046,
            0.13928621064732807
          ],
          "confidence": 0.95,
          "n_rows": 1699,
          "n_text_groups": 1699,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.08416715715126544,
          "harmful_f1_delta_fine_minus_reference": 0.17006824372531704,
          "accuracy_interval": [
            0.061212477928193065,
            0.10653325485579745
          ],
          "harmful_f1_interval": [
            0.1344891071814157,
            0.20434481069783952
          ],
          "confidence": 0.95,
          "n_rows": 1699,
          "n_text_groups": 1699,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.05238375515008831,
          "harmful_f1_delta_fine_minus_reference": 0.06338804635003847,
          "accuracy_interval": [
            0.03472630959387879,
            0.07004120070629782
          ],
          "harmful_f1_interval": [
            0.041071565544565855,
            0.08647652306645585
          ],
          "confidence": 0.95,
          "n_rows": 1699,
          "n_text_groups": 1699,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "wildguard_response",
      "n": 1709,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_response_base.json",
          "input_sha256": "ca9f87126a89853bf29a6bd52394a084b3341a06762ef9e11c325e596c500ad7",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1709,
            "correct": 1276,
            "accuracy": 0.7466354593329433,
            "wilson_95": [
              0.725477288959132,
              0.7666873510656446
            ],
            "macro_f1": 0.5287954881348312,
            "errors": 0,
            "probability_rows": 1709,
            "multiclass_brier": 0.34348986135751897,
            "nll": 0.5185091583323906,
            "ece_10_equal_width": 0.02577899356348743,
            "p50_ms": 120.05620799027383,
            "p95_ms": 410.05266597494483,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1709,
            "per_class": {
              "safe": {
                "support": 1425,
                "negative_support": 284,
                "tp": 1219,
                "fp": 227,
                "fn": 206,
                "tn": 57,
                "precision": 0.843015214384509,
                "recall": 0.8554385964912281,
                "recall_wilson_95": [
                  0.8362241934395617,
                  0.8727417969762595
                ],
                "false_positive_rate": 0.7992957746478874,
                "false_positive_wilson_95": [
                  0.7488589438684632,
                  0.8417439550207431
                ]
              },
              "unsafe": {
                "support": 284,
                "negative_support": 1425,
                "tp": 57,
                "fp": 206,
                "fn": 227,
                "tn": 1219,
                "precision": 0.21673003802281368,
                "recall": 0.2007042253521127,
                "recall_wilson_95": [
                  0.15825604497925705,
                  0.25114105613153687
                ],
                "false_positive_rate": 0.14456140350877192,
                "false_positive_wilson_95": [
                  0.12725820302374047,
                  0.16377580656043844
                ]
              }
            },
            "majority_baseline": 0.8338209479227618,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 57,
              "fp": 206,
              "fn": 227,
              "tn": 1219,
              "precision": 0.21673003802281368,
              "recall": 0.2007042253521127,
              "recall_wilson_95": [
                0.15825604497925705,
                0.25114105613153687
              ],
              "false_positive_rate": 0.14456140350877192,
              "false_positive_wilson_95": [
                0.12725820302374047,
                0.16377580656043844
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 57,
              "fp": 206,
              "fn": 227,
              "tn": 1219,
              "precision": 0.21673003802281368,
              "recall": 0.2007042253521127,
              "recall_wilson_95": [
                0.15825604497925705,
                0.25114105613153687
              ],
              "false_positive_rate": 0.14456140350877192,
              "false_positive_wilson_95": [
                0.12725820302374047,
                0.16377580656043844
              ],
              "f1": 0.20840950639853748,
              "negative_errors": 0
            },
            "harmful_f1": 0.20840950639853748,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.22889039881096718,
              "scored_rows": 1709,
              "excluded_rows": 0,
              "positive_support": 284,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 8,
            "n_retained": 1701,
            "summary": {
              "n": 1701,
              "correct": 1271,
              "accuracy": 0.7472075249853027,
              "wilson_95": [
                0.7260125287412943,
                0.7672884733247586
              ],
              "macro_f1": 0.5267295597484276,
              "errors": 0,
              "probability_rows": 1701,
              "multiclass_brier": 0.3424100636449148,
              "nll": 0.5172277966995233,
              "ece_10_equal_width": 0.02635044091710758,
              "p50_ms": 119.85120899043977,
              "p95_ms": 410.05266597494483,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1701,
              "per_class": {
                "safe": {
                  "support": 1421,
                  "negative_support": 280,
                  "tp": 1216,
                  "fp": 225,
                  "fn": 205,
                  "tn": 55,
                  "precision": 0.8438584316446912,
                  "recall": 0.8557353976073188,
                  "recall_wilson_95": [
                    0.8365073564802625,
                    0.8730452704041205
                  ],
                  "false_positive_rate": 0.8035714285714286,
                  "false_positive_wilson_95": [
                    0.7530612569211522,
                    0.8458646389163228
                  ]
                },
                "unsafe": {
                  "support": 280,
                  "negative_support": 1421,
                  "tp": 55,
                  "fp": 205,
                  "fn": 225,
                  "tn": 1216,
                  "precision": 0.21153846153846154,
                  "recall": 0.19642857142857142,
                  "recall_wilson_95": [
                    0.15413536108367726,
                    0.24693874307884778
                  ],
                  "false_positive_rate": 0.1442646023926812,
                  "false_positive_wilson_95": [
                    0.12695472959587947,
                    0.1634926435197375
                  ]
                }
              },
              "majority_baseline": 0.8353909465020576,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 55,
                "fp": 205,
                "fn": 225,
                "tn": 1216,
                "precision": 0.21153846153846154,
                "recall": 0.19642857142857142,
                "recall_wilson_95": [
                  0.15413536108367726,
                  0.24693874307884778
                ],
                "false_positive_rate": 0.1442646023926812,
                "false_positive_wilson_95": [
                  0.12695472959587947,
                  0.1634926435197375
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 55,
                "fp": 205,
                "fn": 225,
                "tn": 1216,
                "precision": 0.21153846153846154,
                "recall": 0.19642857142857142,
                "recall_wilson_95": [
                  0.15413536108367726,
                  0.24693874307884778
                ],
                "false_positive_rate": 0.1442646023926812,
                "false_positive_wilson_95": [
                  0.12695472959587947,
                  0.1634926435197375
                ],
                "f1": 0.2037037037037037,
                "negative_errors": 0
              },
              "harmful_f1": 0.2037037037037037,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.22654339836927206,
                "scored_rows": 1701,
                "excluded_rows": 0,
                "positive_support": 280,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_response_fine1000.json",
          "input_sha256": "bf8fcb05b79e7d8d8350ff8a9756a367d344a5a89170a270a2ed70afcf8541a8",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1709,
            "correct": 1446,
            "accuracy": 0.8461088355763604,
            "wilson_95": [
              0.8282262653935458,
              0.8624389411892662
            ],
            "macro_f1": 0.720370295483602,
            "errors": 0,
            "probability_rows": 1709,
            "multiclass_brier": 0.2174910165187738,
            "nll": 0.3565188255160757,
            "ece_10_equal_width": 0.03459325662518592,
            "p50_ms": 96.78725013509393,
            "p95_ms": 191.5429588407278,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1709,
            "per_class": {
              "safe": {
                "support": 1425,
                "negative_support": 284,
                "tp": 1296,
                "fp": 134,
                "fn": 129,
                "tn": 150,
                "precision": 0.9062937062937063,
                "recall": 0.9094736842105263,
                "recall_wilson_95": [
                  0.8934543404351649,
                  0.9232912773489053
                ],
                "false_positive_rate": 0.47183098591549294,
                "false_positive_wilson_95": [
                  0.41453554376517077,
                  0.5298783010452569
                ]
              },
              "unsafe": {
                "support": 284,
                "negative_support": 1425,
                "tp": 150,
                "fp": 129,
                "fn": 134,
                "tn": 1296,
                "precision": 0.5376344086021505,
                "recall": 0.528169014084507,
                "recall_wilson_95": [
                  0.4701216989547431,
                  0.5854644562348291
                ],
                "false_positive_rate": 0.09052631578947369,
                "false_positive_wilson_95": [
                  0.07670872265109474,
                  0.10654565956483503
                ]
              }
            },
            "majority_baseline": 0.8338209479227618,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 150,
              "fp": 129,
              "fn": 134,
              "tn": 1296,
              "precision": 0.5376344086021505,
              "recall": 0.528169014084507,
              "recall_wilson_95": [
                0.4701216989547431,
                0.5854644562348291
              ],
              "false_positive_rate": 0.09052631578947369,
              "false_positive_wilson_95": [
                0.07670872265109474,
                0.10654565956483503
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 150,
              "fp": 129,
              "fn": 134,
              "tn": 1296,
              "precision": 0.5376344086021505,
              "recall": 0.528169014084507,
              "recall_wilson_95": [
                0.4701216989547431,
                0.5854644562348291
              ],
              "false_positive_rate": 0.09052631578947369,
              "false_positive_wilson_95": [
                0.07670872265109474,
                0.10654565956483503
              ],
              "f1": 0.5328596802841918,
              "negative_errors": 0
            },
            "harmful_f1": 0.5328596802841918,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.5924538701564802,
              "scored_rows": 1709,
              "excluded_rows": 0,
              "positive_support": 284,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 8,
            "n_retained": 1701,
            "summary": {
              "n": 1701,
              "correct": 1441,
              "accuracy": 0.847148736037625,
              "wilson_95": [
                0.8292673358679696,
                0.8634657003270465
              ],
              "macro_f1": 0.7205089055951304,
              "errors": 0,
              "probability_rows": 1701,
              "multiclass_brier": 0.21620340309852115,
              "nll": 0.355007149617863,
              "ece_10_equal_width": 0.035619209625186776,
              "p50_ms": 96.68812504969537,
              "p95_ms": 188.9174161478877,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1701,
              "per_class": {
                "safe": {
                  "support": 1421,
                  "negative_support": 280,
                  "tp": 1293,
                  "fp": 132,
                  "fn": 128,
                  "tn": 148,
                  "precision": 0.9073684210526316,
                  "recall": 0.9099225897255454,
                  "recall_wilson_95": [
                    0.8939110427610553,
                    0.9237237844598214
                  ],
                  "false_positive_rate": 0.4714285714285714,
                  "false_positive_wilson_95": [
                    0.41374153878680764,
                    0.5298889651343712
                  ]
                },
                "unsafe": {
                  "support": 280,
                  "negative_support": 1421,
                  "tp": 148,
                  "fp": 128,
                  "fn": 132,
                  "tn": 1293,
                  "precision": 0.5362318840579711,
                  "recall": 0.5285714285714286,
                  "recall_wilson_95": [
                    0.4701110348656288,
                    0.5862584612131925
                  ],
                  "false_positive_rate": 0.09007741027445461,
                  "false_positive_wilson_95": [
                    0.07627621554017867,
                    0.10608895723894472
                  ]
                }
              },
              "majority_baseline": 0.8353909465020576,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 148,
                "fp": 128,
                "fn": 132,
                "tn": 1293,
                "precision": 0.5362318840579711,
                "recall": 0.5285714285714286,
                "recall_wilson_95": [
                  0.4701110348656288,
                  0.5862584612131925
                ],
                "false_positive_rate": 0.09007741027445461,
                "false_positive_wilson_95": [
                  0.07627621554017867,
                  0.10608895723894472
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 148,
                "fp": 128,
                "fn": 132,
                "tn": 1293,
                "precision": 0.5362318840579711,
                "recall": 0.5285714285714286,
                "recall_wilson_95": [
                  0.4701110348656288,
                  0.5862584612131925
                ],
                "false_positive_rate": 0.09007741027445461,
                "false_positive_wilson_95": [
                  0.07627621554017867,
                  0.10608895723894472
                ],
                "f1": 0.5323741007194245,
                "negative_errors": 0
              },
              "harmful_f1": 0.5323741007194245,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.5905353863970398,
                "scored_rows": 1701,
                "excluded_rows": 0,
                "positive_support": 280,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/wildguard_response_fine5000.json",
          "input_sha256": "ed4ad42e81af1036eddc7f0e8013cdc4de3365cf66aee0506776e92533310f71",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 1709,
            "correct": 1467,
            "accuracy": 0.8583967232299591,
            "wilson_95": [
              0.8410625000754873,
              0.8741233647291998
            ],
            "macro_f1": 0.7165332419465387,
            "errors": 0,
            "probability_rows": 1709,
            "multiclass_brier": 0.2225650320889409,
            "nll": 0.3679781225493317,
            "ece_10_equal_width": 0.045713516676418944,
            "p50_ms": 97.27608389221132,
            "p95_ms": 191.30183290690184,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 1709,
            "per_class": {
              "safe": {
                "support": 1425,
                "negative_support": 284,
                "tp": 1338,
                "fp": 155,
                "fn": 87,
                "tn": 129,
                "precision": 0.8961821835231079,
                "recall": 0.9389473684210526,
                "recall_wilson_95": [
                  0.9252967980890737,
                  0.9502377073504052
                ],
                "false_positive_rate": 0.545774647887324,
                "false_positive_wilson_95": [
                  0.487641181995174,
                  0.6026863201878812
                ]
              },
              "unsafe": {
                "support": 284,
                "negative_support": 1425,
                "tp": 129,
                "fp": 87,
                "fn": 155,
                "tn": 1338,
                "precision": 0.5972222222222222,
                "recall": 0.45422535211267606,
                "recall_wilson_95": [
                  0.39731367981211885,
                  0.5123588180048261
                ],
                "false_positive_rate": 0.061052631578947365,
                "false_positive_wilson_95": [
                  0.04976229264959482,
                  0.07470320191092626
                ]
              }
            },
            "majority_baseline": 0.8338209479227618,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 129,
              "fp": 87,
              "fn": 155,
              "tn": 1338,
              "precision": 0.5972222222222222,
              "recall": 0.45422535211267606,
              "recall_wilson_95": [
                0.39731367981211885,
                0.5123588180048261
              ],
              "false_positive_rate": 0.061052631578947365,
              "false_positive_wilson_95": [
                0.04976229264959482,
                0.07470320191092626
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 284,
              "negative_support": 1425,
              "tp": 129,
              "fp": 87,
              "fn": 155,
              "tn": 1338,
              "precision": 0.5972222222222222,
              "recall": 0.45422535211267606,
              "recall_wilson_95": [
                0.39731367981211885,
                0.5123588180048261
              ],
              "false_positive_rate": 0.061052631578947365,
              "false_positive_wilson_95": [
                0.04976229264959482,
                0.07470320191092626
              ],
              "f1": 0.516,
              "negative_errors": 0
            },
            "harmful_f1": 0.516,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.5715575260546164,
              "scored_rows": 1709,
              "excluded_rows": 0,
              "positive_support": 284,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {},
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 8,
            "n_retained": 1701,
            "summary": {
              "n": 1701,
              "correct": 1463,
              "accuracy": 0.8600823045267489,
              "wilson_95": [
                0.8427840455275115,
                0.8757578421430174
              ],
              "macro_f1": 0.7181877161425413,
              "errors": 0,
              "probability_rows": 1701,
              "multiclass_brier": 0.22022934044679598,
              "nll": 0.36479337188810296,
              "ece_10_equal_width": 0.04419559082892419,
              "p50_ms": 96.99016599915922,
              "p95_ms": 190.31666708178818,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1701,
              "per_class": {
                "safe": {
                  "support": 1421,
                  "negative_support": 280,
                  "tp": 1335,
                  "fp": 152,
                  "fn": 86,
                  "tn": 128,
                  "precision": 0.8977807666442502,
                  "recall": 0.9394792399718508,
                  "recall_wilson_95": [
                    0.9258566724547461,
                    0.9507320822223997
                  ],
                  "false_positive_rate": 0.5428571428571428,
                  "false_positive_wilson_95": [
                    0.48432081916562253,
                    0.6002334249526091
                  ]
                },
                "unsafe": {
                  "support": 280,
                  "negative_support": 1421,
                  "tp": 128,
                  "fp": 86,
                  "fn": 152,
                  "tn": 1335,
                  "precision": 0.5981308411214953,
                  "recall": 0.45714285714285713,
                  "recall_wilson_95": [
                    0.3997665750473909,
                    0.5156791808343775
                  ],
                  "false_positive_rate": 0.06052076002814919,
                  "false_positive_wilson_95": [
                    0.04926791777760029,
                    0.07414332754525388
                  ]
                }
              },
              "majority_baseline": 0.8353909465020576,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 128,
                "fp": 86,
                "fn": 152,
                "tn": 1335,
                "precision": 0.5981308411214953,
                "recall": 0.45714285714285713,
                "recall_wilson_95": [
                  0.3997665750473909,
                  0.5156791808343775
                ],
                "false_positive_rate": 0.06052076002814919,
                "false_positive_wilson_95": [
                  0.04926791777760029,
                  0.07414332754525388
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 280,
                "negative_support": 1421,
                "tp": 128,
                "fp": 86,
                "fn": 152,
                "tn": 1335,
                "precision": 0.5981308411214953,
                "recall": 0.45714285714285713,
                "recall_wilson_95": [
                  0.3997665750473909,
                  0.5156791808343775
                ],
                "false_positive_rate": 0.06052076002814919,
                "false_positive_wilson_95": [
                  0.04926791777760029,
                  0.07414332754525388
                ],
                "f1": 0.5182186234817814,
                "negative_errors": 0
              },
              "harmful_f1": 0.5182186234817814,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.5711456922082032,
                "scored_rows": 1701,
                "excluded_rows": 0,
                "positive_support": 280,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {},
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.09947337624341712,
          "harmful_f1_delta_fine_minus_reference": 0.32445017388565434,
          "accuracy_interval": [
            0.080163838502048,
            0.11878291398478646
          ],
          "harmful_f1_interval": [
            0.27208413972042994,
            0.37515820676990924
          ],
          "confidence": 0.95,
          "n_rows": 1709,
          "n_text_groups": 1709,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.1117612638970158,
          "harmful_f1_delta_fine_minus_reference": 0.3075904936014625,
          "accuracy_interval": [
            0.09303686366296082,
            0.13048566413107077
          ],
          "harmful_f1_interval": [
            0.25248430541226496,
            0.36115676750655
          ],
          "confidence": 0.95,
          "n_rows": 1709,
          "n_text_groups": 1709,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.012287887653598673,
          "harmful_f1_delta_fine_minus_reference": -0.016859680284191825,
          "accuracy_interval": [
            -0.002340550029256927,
            0.02691632533645405
          ],
          "harmful_f1_interval": [
            -0.061999817551541714,
            0.028346855960038893
          ],
          "confidence": 0.95,
          "n_rows": 1709,
          "n_text_groups": 1709,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "openai_known",
      "n": 859,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_known_base.json",
          "input_sha256": "a47314fad877eb18093f1b2540b61f04f3df527b95c6d5020bc7290d0ea9c759",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 859,
            "correct": 516,
            "accuracy": 0.6006984866123399,
            "wilson_95": [
              0.5675686499803945,
              0.6329316832285026
            ],
            "macro_f1": 0.6004819107204118,
            "errors": 0,
            "probability_rows": 859,
            "multiclass_brier": 0.4868927243538999,
            "nll": 0.6855101297281724,
            "ece_10_equal_width": 0.05699522700814902,
            "p50_ms": 55.06695806980133,
            "p95_ms": 253.79179185256362,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 859,
            "per_class": {
              "safe": {
                "support": 337,
                "negative_support": 522,
                "tp": 268,
                "fp": 274,
                "fn": 69,
                "tn": 248,
                "precision": 0.4944649446494465,
                "recall": 0.7952522255192879,
                "recall_wilson_95": [
                  0.7489570639651255,
                  0.8348920977200076
                ],
                "false_positive_rate": 0.524904214559387,
                "false_positive_wilson_95": [
                  0.4820392502866341,
                  0.5674053105479381
                ]
              },
              "unsafe": {
                "support": 522,
                "negative_support": 337,
                "tp": 248,
                "fp": 69,
                "fn": 274,
                "tn": 268,
                "precision": 0.7823343848580442,
                "recall": 0.47509578544061304,
                "recall_wilson_95": [
                  0.4325946894520619,
                  0.5179607497133659
                ],
                "false_positive_rate": 0.20474777448071216,
                "false_positive_wilson_95": [
                  0.16510790227999256,
                  0.2510429360348746
                ]
              }
            },
            "majority_baseline": 0.6076833527357393,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 248,
              "fp": 69,
              "fn": 274,
              "tn": 268,
              "precision": 0.7823343848580442,
              "recall": 0.47509578544061304,
              "recall_wilson_95": [
                0.4325946894520619,
                0.5179607497133659
              ],
              "false_positive_rate": 0.20474777448071216,
              "false_positive_wilson_95": [
                0.16510790227999256,
                0.2510429360348746
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 248,
              "fp": 69,
              "fn": 274,
              "tn": 268,
              "precision": 0.7823343848580442,
              "recall": 0.47509578544061304,
              "recall_wilson_95": [
                0.4325946894520619,
                0.5179607497133659
              ],
              "false_positive_rate": 0.20474777448071216,
              "false_positive_wilson_95": [
                0.16510790227999256,
                0.2510429360348746
              ],
              "f1": 0.5911799761620977,
              "negative_errors": 0
            },
            "harmful_f1": 0.5911799761620977,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.7455214452688965,
              "scored_rows": 859,
              "excluded_rows": 0,
              "positive_support": 522,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 76,
                "recall": 0.4691358024691358,
                "recall_wilson_95": [
                  0.3938947766491506,
                  0.5458066703164607
                ]
              },
              "H2": {
                "support": 41,
                "tp": 17,
                "recall": 0.4146341463414634,
                "recall_wilson_95": [
                  0.27756549454252133,
                  0.5663289686596986
                ]
              },
              "HR": {
                "support": 76,
                "tp": 50,
                "recall": 0.6578947368421053,
                "recall_wilson_95": [
                  0.5459589781543774,
                  0.7546367318250531
                ]
              },
              "S": {
                "support": 237,
                "tp": 127,
                "recall": 0.5358649789029536,
                "recall_wilson_95": [
                  0.472306026744132,
                  0.5982798270504858
                ]
              },
              "S3": {
                "support": 85,
                "tp": 48,
                "recall": 0.5647058823529412,
                "recall_wilson_95": [
                  0.4587738410724954,
                  0.6650422257280392
                ]
              },
              "SH": {
                "support": 51,
                "tp": 10,
                "recall": 0.19607843137254902,
                "recall_wilson_95": [
                  0.11015342512466689,
                  0.3245807948136561
                ]
              },
              "V": {
                "support": 94,
                "tp": 37,
                "recall": 0.39361702127659576,
                "recall_wilson_95": [
                  0.3008989325900031,
                  0.4946887432167085
                ]
              },
              "V2": {
                "support": 24,
                "tp": 11,
                "recall": 0.4583333333333333,
                "recall_wilson_95": [
                  0.2789133373121099,
                  0.6492513464108051
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 859,
            "summary": {
              "n": 859,
              "correct": 516,
              "accuracy": 0.6006984866123399,
              "wilson_95": [
                0.5675686499803945,
                0.6329316832285026
              ],
              "macro_f1": 0.6004819107204118,
              "errors": 0,
              "probability_rows": 859,
              "multiclass_brier": 0.4868927243538999,
              "nll": 0.6855101297281724,
              "ece_10_equal_width": 0.05699522700814902,
              "p50_ms": 55.06695806980133,
              "p95_ms": 253.79179185256362,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 859,
              "per_class": {
                "safe": {
                  "support": 337,
                  "negative_support": 522,
                  "tp": 268,
                  "fp": 274,
                  "fn": 69,
                  "tn": 248,
                  "precision": 0.4944649446494465,
                  "recall": 0.7952522255192879,
                  "recall_wilson_95": [
                    0.7489570639651255,
                    0.8348920977200076
                  ],
                  "false_positive_rate": 0.524904214559387,
                  "false_positive_wilson_95": [
                    0.4820392502866341,
                    0.5674053105479381
                  ]
                },
                "unsafe": {
                  "support": 522,
                  "negative_support": 337,
                  "tp": 248,
                  "fp": 69,
                  "fn": 274,
                  "tn": 268,
                  "precision": 0.7823343848580442,
                  "recall": 0.47509578544061304,
                  "recall_wilson_95": [
                    0.4325946894520619,
                    0.5179607497133659
                  ],
                  "false_positive_rate": 0.20474777448071216,
                  "false_positive_wilson_95": [
                    0.16510790227999256,
                    0.2510429360348746
                  ]
                }
              },
              "majority_baseline": 0.6076833527357393,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 248,
                "fp": 69,
                "fn": 274,
                "tn": 268,
                "precision": 0.7823343848580442,
                "recall": 0.47509578544061304,
                "recall_wilson_95": [
                  0.4325946894520619,
                  0.5179607497133659
                ],
                "false_positive_rate": 0.20474777448071216,
                "false_positive_wilson_95": [
                  0.16510790227999256,
                  0.2510429360348746
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 248,
                "fp": 69,
                "fn": 274,
                "tn": 268,
                "precision": 0.7823343848580442,
                "recall": 0.47509578544061304,
                "recall_wilson_95": [
                  0.4325946894520619,
                  0.5179607497133659
                ],
                "false_positive_rate": 0.20474777448071216,
                "false_positive_wilson_95": [
                  0.16510790227999256,
                  0.2510429360348746
                ],
                "f1": 0.5911799761620977,
                "negative_errors": 0
              },
              "harmful_f1": 0.5911799761620977,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.7455214452688965,
                "scored_rows": 859,
                "excluded_rows": 0,
                "positive_support": 522,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 76,
                  "recall": 0.4691358024691358,
                  "recall_wilson_95": [
                    0.3938947766491506,
                    0.5458066703164607
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 17,
                  "recall": 0.4146341463414634,
                  "recall_wilson_95": [
                    0.27756549454252133,
                    0.5663289686596986
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 50,
                  "recall": 0.6578947368421053,
                  "recall_wilson_95": [
                    0.5459589781543774,
                    0.7546367318250531
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 127,
                  "recall": 0.5358649789029536,
                  "recall_wilson_95": [
                    0.472306026744132,
                    0.5982798270504858
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 48,
                  "recall": 0.5647058823529412,
                  "recall_wilson_95": [
                    0.4587738410724954,
                    0.6650422257280392
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 10,
                  "recall": 0.19607843137254902,
                  "recall_wilson_95": [
                    0.11015342512466689,
                    0.3245807948136561
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 37,
                  "recall": 0.39361702127659576,
                  "recall_wilson_95": [
                    0.3008989325900031,
                    0.4946887432167085
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 11,
                  "recall": 0.4583333333333333,
                  "recall_wilson_95": [
                    0.2789133373121099,
                    0.6492513464108051
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_known_fine1000.json",
          "input_sha256": "49a214dbbc472d5ecad6d3fc1a721dace9b7e6723773f53fe1614d2fbdacfd7b",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 859,
            "correct": 654,
            "accuracy": 0.7613504074505238,
            "wilson_95": [
              0.7317213787980371,
              0.788652318374187
            ],
            "macro_f1": 0.7356157205764672,
            "errors": 0,
            "probability_rows": 859,
            "multiclass_brier": 0.3372030724373448,
            "nll": 0.5163499024067627,
            "ece_10_equal_width": 0.029286303721199444,
            "p50_ms": 59.98320784419775,
            "p95_ms": 130.56045793928206,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 859,
            "per_class": {
              "safe": {
                "support": 337,
                "negative_support": 522,
                "tp": 193,
                "fp": 61,
                "fn": 144,
                "tn": 461,
                "precision": 0.7598425196850394,
                "recall": 0.5727002967359051,
                "recall_wilson_95": [
                  0.5193572967952431,
                  0.624404557087026
                ],
                "false_positive_rate": 0.11685823754789272,
                "false_positive_wilson_95": [
                  0.09205714321005112,
                  0.14725730548883817
                ]
              },
              "unsafe": {
                "support": 522,
                "negative_support": 337,
                "tp": 461,
                "fp": 144,
                "fn": 61,
                "tn": 193,
                "precision": 0.7619834710743801,
                "recall": 0.8831417624521073,
                "recall_wilson_95": [
                  0.8527426945111618,
                  0.9079428567899488
                ],
                "false_positive_rate": 0.42729970326409494,
                "false_positive_wilson_95": [
                  0.3755954429129742,
                  0.480642703204757
                ]
              }
            },
            "majority_baseline": 0.6076833527357393,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 461,
              "fp": 144,
              "fn": 61,
              "tn": 193,
              "precision": 0.7619834710743801,
              "recall": 0.8831417624521073,
              "recall_wilson_95": [
                0.8527426945111618,
                0.9079428567899488
              ],
              "false_positive_rate": 0.42729970326409494,
              "false_positive_wilson_95": [
                0.3755954429129742,
                0.480642703204757
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 461,
              "fp": 144,
              "fn": 61,
              "tn": 193,
              "precision": 0.7619834710743801,
              "recall": 0.8831417624521073,
              "recall_wilson_95": [
                0.8527426945111618,
                0.9079428567899488
              ],
              "false_positive_rate": 0.42729970326409494,
              "false_positive_wilson_95": [
                0.3755954429129742,
                0.480642703204757
              ],
              "f1": 0.8181011535048802,
              "negative_errors": 0
            },
            "harmful_f1": 0.8181011535048802,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8469951788669784,
              "scored_rows": 859,
              "excluded_rows": 0,
              "positive_support": 522,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 148,
                "recall": 0.9135802469135802,
                "recall_wilson_95": [
                  0.8601761009899326,
                  0.9478245096708747
                ]
              },
              "H2": {
                "support": 41,
                "tp": 40,
                "recall": 0.975609756097561,
                "recall_wilson_95": [
                  0.8740493794422576,
                  0.9956814684310874
                ]
              },
              "HR": {
                "support": 76,
                "tp": 69,
                "recall": 0.9078947368421053,
                "recall_wilson_95": [
                  0.8218729215106378,
                  0.9546659959362239
                ]
              },
              "S": {
                "support": 237,
                "tp": 223,
                "recall": 0.9409282700421941,
                "recall_wilson_95": [
                  0.9033011630807447,
                  0.9644896276883811
                ]
              },
              "S3": {
                "support": 85,
                "tp": 82,
                "recall": 0.9647058823529412,
                "recall_wilson_95": [
                  0.9012995216039354,
                  0.987924958145358
                ]
              },
              "SH": {
                "support": 51,
                "tp": 31,
                "recall": 0.6078431372549019,
                "recall_wilson_95": [
                  0.4708515898394834,
                  0.7297265901824019
                ]
              },
              "V": {
                "support": 94,
                "tp": 84,
                "recall": 0.8936170212765957,
                "recall_wilson_95": [
                  0.8151444897581666,
                  0.9411811097570002
                ]
              },
              "V2": {
                "support": 24,
                "tp": 16,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 859,
            "summary": {
              "n": 859,
              "correct": 654,
              "accuracy": 0.7613504074505238,
              "wilson_95": [
                0.7317213787980371,
                0.788652318374187
              ],
              "macro_f1": 0.7356157205764672,
              "errors": 0,
              "probability_rows": 859,
              "multiclass_brier": 0.3372030724373448,
              "nll": 0.5163499024067627,
              "ece_10_equal_width": 0.029286303721199444,
              "p50_ms": 59.98320784419775,
              "p95_ms": 130.56045793928206,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 859,
              "per_class": {
                "safe": {
                  "support": 337,
                  "negative_support": 522,
                  "tp": 193,
                  "fp": 61,
                  "fn": 144,
                  "tn": 461,
                  "precision": 0.7598425196850394,
                  "recall": 0.5727002967359051,
                  "recall_wilson_95": [
                    0.5193572967952431,
                    0.624404557087026
                  ],
                  "false_positive_rate": 0.11685823754789272,
                  "false_positive_wilson_95": [
                    0.09205714321005112,
                    0.14725730548883817
                  ]
                },
                "unsafe": {
                  "support": 522,
                  "negative_support": 337,
                  "tp": 461,
                  "fp": 144,
                  "fn": 61,
                  "tn": 193,
                  "precision": 0.7619834710743801,
                  "recall": 0.8831417624521073,
                  "recall_wilson_95": [
                    0.8527426945111618,
                    0.9079428567899488
                  ],
                  "false_positive_rate": 0.42729970326409494,
                  "false_positive_wilson_95": [
                    0.3755954429129742,
                    0.480642703204757
                  ]
                }
              },
              "majority_baseline": 0.6076833527357393,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 461,
                "fp": 144,
                "fn": 61,
                "tn": 193,
                "precision": 0.7619834710743801,
                "recall": 0.8831417624521073,
                "recall_wilson_95": [
                  0.8527426945111618,
                  0.9079428567899488
                ],
                "false_positive_rate": 0.42729970326409494,
                "false_positive_wilson_95": [
                  0.3755954429129742,
                  0.480642703204757
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 461,
                "fp": 144,
                "fn": 61,
                "tn": 193,
                "precision": 0.7619834710743801,
                "recall": 0.8831417624521073,
                "recall_wilson_95": [
                  0.8527426945111618,
                  0.9079428567899488
                ],
                "false_positive_rate": 0.42729970326409494,
                "false_positive_wilson_95": [
                  0.3755954429129742,
                  0.480642703204757
                ],
                "f1": 0.8181011535048802,
                "negative_errors": 0
              },
              "harmful_f1": 0.8181011535048802,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8469951788669784,
                "scored_rows": 859,
                "excluded_rows": 0,
                "positive_support": 522,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 148,
                  "recall": 0.9135802469135802,
                  "recall_wilson_95": [
                    0.8601761009899326,
                    0.9478245096708747
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 40,
                  "recall": 0.975609756097561,
                  "recall_wilson_95": [
                    0.8740493794422576,
                    0.9956814684310874
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 69,
                  "recall": 0.9078947368421053,
                  "recall_wilson_95": [
                    0.8218729215106378,
                    0.9546659959362239
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 223,
                  "recall": 0.9409282700421941,
                  "recall_wilson_95": [
                    0.9033011630807447,
                    0.9644896276883811
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 82,
                  "recall": 0.9647058823529412,
                  "recall_wilson_95": [
                    0.9012995216039354,
                    0.987924958145358
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 31,
                  "recall": 0.6078431372549019,
                  "recall_wilson_95": [
                    0.4708515898394834,
                    0.7297265901824019
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 84,
                  "recall": 0.8936170212765957,
                  "recall_wilson_95": [
                    0.8151444897581666,
                    0.9411811097570002
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_known_fine5000.json",
          "input_sha256": "446bfb23983b6a0130268b9cf415ceefeda832bd54699435f79ed7fa47e53d70",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 859,
            "correct": 659,
            "accuracy": 0.7671711292200233,
            "wilson_95": [
              0.737756709453349,
              0.7942066023552274
            ],
            "macro_f1": 0.7516106272482275,
            "errors": 0,
            "probability_rows": 859,
            "multiclass_brier": 0.3421863494233844,
            "nll": 0.5335324801592212,
            "ece_10_equal_width": 0.08894597616921177,
            "p50_ms": 60.81070797517896,
            "p95_ms": 130.6580000091344,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 859,
            "per_class": {
              "safe": {
                "support": 337,
                "negative_support": 522,
                "tp": 222,
                "fp": 85,
                "fn": 115,
                "tn": 437,
                "precision": 0.7231270358306189,
                "recall": 0.658753709198813,
                "recall_wilson_95": [
                  0.6065979933396797,
                  0.7073309528930303
                ],
                "false_positive_rate": 0.16283524904214558,
                "false_positive_wilson_95": [
                  0.13364504060925875,
                  0.19695167424576382
                ]
              },
              "unsafe": {
                "support": 522,
                "negative_support": 337,
                "tp": 437,
                "fp": 115,
                "fn": 85,
                "tn": 222,
                "precision": 0.7916666666666666,
                "recall": 0.8371647509578544,
                "recall_wilson_95": [
                  0.8030483257542361,
                  0.8663549593907413
                ],
                "false_positive_rate": 0.34124629080118696,
                "false_positive_wilson_95": [
                  0.29266904710696995,
                  0.39340200666032055
                ]
              }
            },
            "majority_baseline": 0.6076833527357393,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 437,
              "fp": 115,
              "fn": 85,
              "tn": 222,
              "precision": 0.7916666666666666,
              "recall": 0.8371647509578544,
              "recall_wilson_95": [
                0.8030483257542361,
                0.8663549593907413
              ],
              "false_positive_rate": 0.34124629080118696,
              "false_positive_wilson_95": [
                0.29266904710696995,
                0.39340200666032055
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 522,
              "negative_support": 337,
              "tp": 437,
              "fp": 115,
              "fn": 85,
              "tn": 222,
              "precision": 0.7916666666666666,
              "recall": 0.8371647509578544,
              "recall_wilson_95": [
                0.8030483257542361,
                0.8663549593907413
              ],
              "false_positive_rate": 0.34124629080118696,
              "false_positive_wilson_95": [
                0.29266904710696995,
                0.39340200666032055
              ],
              "f1": 0.813780260707635,
              "negative_errors": 0
            },
            "harmful_f1": 0.813780260707635,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8643477709945026,
              "scored_rows": 859,
              "excluded_rows": 0,
              "positive_support": 522,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 145,
                "recall": 0.8950617283950617,
                "recall_wilson_95": [
                  0.8383776426456437,
                  0.9334438361945305
                ]
              },
              "H2": {
                "support": 41,
                "tp": 41,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ]
              },
              "HR": {
                "support": 76,
                "tp": 65,
                "recall": 0.8552631578947368,
                "recall_wilson_95": [
                  0.7591255101452948,
                  0.9172148373084233
                ]
              },
              "S": {
                "support": 237,
                "tp": 206,
                "recall": 0.869198312236287,
                "recall_wilson_95": [
                  0.8203200961821225,
                  0.9062989869977675
                ]
              },
              "S3": {
                "support": 85,
                "tp": 78,
                "recall": 0.9176470588235294,
                "recall_wilson_95": [
                  0.839640154666247,
                  0.9595362765008394
                ]
              },
              "SH": {
                "support": 51,
                "tp": 33,
                "recall": 0.6470588235294118,
                "recall_wilson_95": [
                  0.5098605845450523,
                  0.7636551154847915
                ]
              },
              "V": {
                "support": 94,
                "tp": 84,
                "recall": 0.8936170212765957,
                "recall_wilson_95": [
                  0.8151444897581666,
                  0.9411811097570002
                ]
              },
              "V2": {
                "support": 24,
                "tp": 16,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 859,
            "summary": {
              "n": 859,
              "correct": 659,
              "accuracy": 0.7671711292200233,
              "wilson_95": [
                0.737756709453349,
                0.7942066023552274
              ],
              "macro_f1": 0.7516106272482275,
              "errors": 0,
              "probability_rows": 859,
              "multiclass_brier": 0.3421863494233844,
              "nll": 0.5335324801592212,
              "ece_10_equal_width": 0.08894597616921177,
              "p50_ms": 60.81070797517896,
              "p95_ms": 130.6580000091344,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 859,
              "per_class": {
                "safe": {
                  "support": 337,
                  "negative_support": 522,
                  "tp": 222,
                  "fp": 85,
                  "fn": 115,
                  "tn": 437,
                  "precision": 0.7231270358306189,
                  "recall": 0.658753709198813,
                  "recall_wilson_95": [
                    0.6065979933396797,
                    0.7073309528930303
                  ],
                  "false_positive_rate": 0.16283524904214558,
                  "false_positive_wilson_95": [
                    0.13364504060925875,
                    0.19695167424576382
                  ]
                },
                "unsafe": {
                  "support": 522,
                  "negative_support": 337,
                  "tp": 437,
                  "fp": 115,
                  "fn": 85,
                  "tn": 222,
                  "precision": 0.7916666666666666,
                  "recall": 0.8371647509578544,
                  "recall_wilson_95": [
                    0.8030483257542361,
                    0.8663549593907413
                  ],
                  "false_positive_rate": 0.34124629080118696,
                  "false_positive_wilson_95": [
                    0.29266904710696995,
                    0.39340200666032055
                  ]
                }
              },
              "majority_baseline": 0.6076833527357393,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 437,
                "fp": 115,
                "fn": 85,
                "tn": 222,
                "precision": 0.7916666666666666,
                "recall": 0.8371647509578544,
                "recall_wilson_95": [
                  0.8030483257542361,
                  0.8663549593907413
                ],
                "false_positive_rate": 0.34124629080118696,
                "false_positive_wilson_95": [
                  0.29266904710696995,
                  0.39340200666032055
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 522,
                "negative_support": 337,
                "tp": 437,
                "fp": 115,
                "fn": 85,
                "tn": 222,
                "precision": 0.7916666666666666,
                "recall": 0.8371647509578544,
                "recall_wilson_95": [
                  0.8030483257542361,
                  0.8663549593907413
                ],
                "false_positive_rate": 0.34124629080118696,
                "false_positive_wilson_95": [
                  0.29266904710696995,
                  0.39340200666032055
                ],
                "f1": 0.813780260707635,
                "negative_errors": 0
              },
              "harmful_f1": 0.813780260707635,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8643477709945026,
                "scored_rows": 859,
                "excluded_rows": 0,
                "positive_support": 522,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 145,
                  "recall": 0.8950617283950617,
                  "recall_wilson_95": [
                    0.8383776426456437,
                    0.9334438361945305
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 41,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 65,
                  "recall": 0.8552631578947368,
                  "recall_wilson_95": [
                    0.7591255101452948,
                    0.9172148373084233
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 206,
                  "recall": 0.869198312236287,
                  "recall_wilson_95": [
                    0.8203200961821225,
                    0.9062989869977675
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 78,
                  "recall": 0.9176470588235294,
                  "recall_wilson_95": [
                    0.839640154666247,
                    0.9595362765008394
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 33,
                  "recall": 0.6470588235294118,
                  "recall_wilson_95": [
                    0.5098605845450523,
                    0.7636551154847915
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 84,
                  "recall": 0.8936170212765957,
                  "recall_wilson_95": [
                    0.8151444897581666,
                    0.9411811097570002
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": 0.16065192083818391,
          "harmful_f1_delta_fine_minus_reference": 0.2269211773427825,
          "accuracy_interval": [
            0.1212085935019579,
            0.20023282887077998
          ],
          "harmful_f1_interval": [
            0.18713442570665448,
            0.2667601886475129
          ],
          "confidence": 0.95,
          "n_rows": 859,
          "n_text_groups": 848,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.16647264260768335,
          "harmful_f1_delta_fine_minus_reference": 0.22260028454553726,
          "accuracy_interval": [
            0.1268917345750873,
            0.2062572421784472
          ],
          "harmful_f1_interval": [
            0.18093576068613554,
            0.26385443469430614
          ],
          "confidence": 0.95,
          "n_rows": 859,
          "n_text_groups": 848,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.005820721769499437,
          "harmful_f1_delta_fine_minus_reference": -0.004320892797245235,
          "accuracy_interval": [
            -0.017421602787456417,
            0.027972027972028024
          ],
          "harmful_f1_interval": [
            -0.022674334770626015,
            0.013036659201459255
          ],
          "confidence": 0.95,
          "n_rows": 859,
          "n_text_groups": 848,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "openai_categories",
      "n": 9298,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_categories_base.json",
          "input_sha256": "cf232b429408dcd8f3ddc820a5a9570cafffa0d171c0dbf8a3a03d7afafef16e",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 9298,
            "correct": 5848,
            "accuracy": 0.6289524628952463,
            "wilson_95": [
              0.6190818712644978,
              0.6387165453840723
            ],
            "macro_f1": 0.4933706593444915,
            "errors": 0,
            "probability_rows": 9298,
            "multiclass_brier": 0.455494301696372,
            "nll": 0.651483254799129,
            "ece_10_equal_width": 0.07492664050383899,
            "p50_ms": 31.325000105425715,
            "p95_ms": 202.31587486341596,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 9298,
            "per_class": {
              "safe": {
                "support": 8528,
                "negative_support": 770,
                "tp": 5329,
                "fp": 251,
                "fn": 3199,
                "tn": 519,
                "precision": 0.9550179211469534,
                "recall": 0.6248827392120075,
                "recall_wilson_95": [
                  0.6145530682636524,
                  0.6350999533355579
                ],
                "false_positive_rate": 0.32597402597402597,
                "false_positive_wilson_95": [
                  0.29380092801282226,
                  0.35987490319888077
                ]
              },
              "unsafe": {
                "support": 770,
                "negative_support": 8528,
                "tp": 519,
                "fp": 3199,
                "fn": 251,
                "tn": 5329,
                "precision": 0.1395911780527165,
                "recall": 0.674025974025974,
                "recall_wilson_95": [
                  0.6401250968011191,
                  0.7061990719871777
                ],
                "false_positive_rate": 0.3751172607879925,
                "false_positive_wilson_95": [
                  0.36490004666444215,
                  0.38544693173634764
                ]
              }
            },
            "majority_baseline": 0.9171864917186492,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 519,
              "fp": 3199,
              "fn": 251,
              "tn": 5329,
              "precision": 0.1395911780527165,
              "recall": 0.674025974025974,
              "recall_wilson_95": [
                0.6401250968011191,
                0.7061990719871777
              ],
              "false_positive_rate": 0.3751172607879925,
              "false_positive_wilson_95": [
                0.36490004666444215,
                0.38544693173634764
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 519,
              "fp": 3199,
              "fn": 251,
              "tn": 5329,
              "precision": 0.1395911780527165,
              "recall": 0.674025974025974,
              "recall_wilson_95": [
                0.6401250968011191,
                0.7061990719871777
              ],
              "false_positive_rate": 0.3751172607879925,
              "false_positive_wilson_95": [
                0.36490004666444215,
                0.38544693173634764
              ],
              "f1": 0.23128342245989306,
              "negative_errors": 0
            },
            "harmful_f1": 0.23128342245989306,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.11551582971319406,
              "scored_rows": 9298,
              "excluded_rows": 0,
              "positive_support": 770,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 119,
                "recall": 0.7345679012345679,
                "recall_wilson_95": [
                  0.6617113535920193,
                  0.7965576494693342
                ]
              },
              "H2": {
                "support": 41,
                "tp": 24,
                "recall": 0.5853658536585366,
                "recall_wilson_95": [
                  0.4336710313403013,
                  0.7224345054574786
                ]
              },
              "HR": {
                "support": 76,
                "tp": 50,
                "recall": 0.6578947368421053,
                "recall_wilson_95": [
                  0.5459589781543774,
                  0.7546367318250531
                ]
              },
              "S": {
                "support": 237,
                "tp": 187,
                "recall": 0.7890295358649789,
                "recall_wilson_95": [
                  0.7326860529664931,
                  0.8361528864371921
                ]
              },
              "S3": {
                "support": 85,
                "tp": 65,
                "recall": 0.7647058823529411,
                "recall_wilson_95": [
                  0.6643156605319014,
                  0.8422046127430125
                ]
              },
              "SH": {
                "support": 51,
                "tp": 14,
                "recall": 0.27450980392156865,
                "recall_wilson_95": [
                  0.17114275234644882,
                  0.4094665076077909
                ]
              },
              "V": {
                "support": 94,
                "tp": 50,
                "recall": 0.5319148936170213,
                "recall_wilson_95": [
                  0.4317826374815637,
                  0.6295410597764227
                ]
              },
              "V2": {
                "support": 24,
                "tp": 10,
                "recall": 0.4166666666666667,
                "recall_wilson_95": [
                  0.24467602609811764,
                  0.6116533413477124
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {
            "H": {
              "n": 771,
              "correct": 461,
              "accuracy": 0.5979247730220493,
              "wilson_95": [
                0.5629120741282438,
                0.6319665017159313
              ],
              "macro_f1": 0.5612176709894403,
              "errors": 0,
              "probability_rows": 771,
              "multiclass_brier": 0.5034585193514916,
              "nll": 0.7127450565979375,
              "ece_10_equal_width": 0.07923281452658884,
              "p50_ms": 33.268958097323775,
              "p95_ms": 210.0077080540359,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 771,
              "per_class": {
                "safe": {
                  "support": 609,
                  "negative_support": 162,
                  "tp": 342,
                  "fp": 43,
                  "fn": 267,
                  "tn": 119,
                  "precision": 0.8883116883116883,
                  "recall": 0.5615763546798029,
                  "recall_wilson_95": [
                    0.5219036026616696,
                    0.60047715164357
                  ],
                  "false_positive_rate": 0.2654320987654321,
                  "false_positive_wilson_95": [
                    0.20344235053066584,
                    0.33828864640798073
                  ]
                },
                "unsafe": {
                  "support": 162,
                  "negative_support": 609,
                  "tp": 119,
                  "fp": 267,
                  "fn": 43,
                  "tn": 342,
                  "precision": 0.3082901554404145,
                  "recall": 0.7345679012345679,
                  "recall_wilson_95": [
                    0.6617113535920193,
                    0.7965576494693342
                  ],
                  "false_positive_rate": 0.43842364532019706,
                  "false_positive_wilson_95": [
                    0.39952284835643004,
                    0.4780963973383304
                  ]
                }
              },
              "majority_baseline": 0.7898832684824902,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 119,
                "fp": 267,
                "fn": 43,
                "tn": 342,
                "precision": 0.3082901554404145,
                "recall": 0.7345679012345679,
                "recall_wilson_95": [
                  0.6617113535920193,
                  0.7965576494693342
                ],
                "false_positive_rate": 0.43842364532019706,
                "false_positive_wilson_95": [
                  0.39952284835643004,
                  0.4780963973383304
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 119,
                "fp": 267,
                "fn": 43,
                "tn": 342,
                "precision": 0.3082901554404145,
                "recall": 0.7345679012345679,
                "recall_wilson_95": [
                  0.6617113535920193,
                  0.7965576494693342
                ],
                "false_positive_rate": 0.43842364532019706,
                "false_positive_wilson_95": [
                  0.39952284835643004,
                  0.4780963973383304
                ],
                "f1": 0.4343065693430657,
                "negative_errors": 0
              },
              "harmful_f1": 0.4343065693430657,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.24851296007512683,
                "scored_rows": 771,
                "excluded_rows": 0,
                "positive_support": 162,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 119,
                  "recall": 0.7345679012345679,
                  "recall_wilson_95": [
                    0.6617113535920193,
                    0.7965576494693342
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "H2": {
              "n": 761,
              "correct": 386,
              "accuracy": 0.507227332457293,
              "wilson_95": [
                0.4717597204926377,
                0.5426223450624393
              ],
              "macro_f1": 0.38612794352054414,
              "errors": 0,
              "probability_rows": 761,
              "multiclass_brier": 0.5453099349598581,
              "nll": 0.7570545585828292,
              "ece_10_equal_width": 0.16389944081807323,
              "p50_ms": 32.13095781393349,
              "p95_ms": 209.71687487326562,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 761,
              "per_class": {
                "safe": {
                  "support": 720,
                  "negative_support": 41,
                  "tp": 362,
                  "fp": 17,
                  "fn": 358,
                  "tn": 24,
                  "precision": 0.9551451187335093,
                  "recall": 0.5027777777777778,
                  "recall_wilson_95": [
                    0.46633886285094406,
                    0.5391872091241497
                  ],
                  "false_positive_rate": 0.4146341463414634,
                  "false_positive_wilson_95": [
                    0.27756549454252133,
                    0.5663289686596986
                  ]
                },
                "unsafe": {
                  "support": 41,
                  "negative_support": 720,
                  "tp": 24,
                  "fp": 358,
                  "fn": 17,
                  "tn": 362,
                  "precision": 0.06282722513089005,
                  "recall": 0.5853658536585366,
                  "recall_wilson_95": [
                    0.4336710313403013,
                    0.7224345054574786
                  ],
                  "false_positive_rate": 0.49722222222222223,
                  "false_positive_wilson_95": [
                    0.46081279087585036,
                    0.533661137149056
                  ]
                }
              },
              "majority_baseline": 0.9461235216819974,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 24,
                "fp": 358,
                "fn": 17,
                "tn": 362,
                "precision": 0.06282722513089005,
                "recall": 0.5853658536585366,
                "recall_wilson_95": [
                  0.4336710313403013,
                  0.7224345054574786
                ],
                "false_positive_rate": 0.49722222222222223,
                "false_positive_wilson_95": [
                  0.46081279087585036,
                  0.533661137149056
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 24,
                "fp": 358,
                "fn": 17,
                "tn": 362,
                "precision": 0.06282722513089005,
                "recall": 0.5853658536585366,
                "recall_wilson_95": [
                  0.4336710313403013,
                  0.7224345054574786
                ],
                "false_positive_rate": 0.49722222222222223,
                "false_positive_wilson_95": [
                  0.46081279087585036,
                  0.533661137149056
                ],
                "f1": 0.11347517730496454,
                "negative_errors": 0
              },
              "harmful_f1": 0.11347517730496454,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.05667493763607429,
                "scored_rows": 761,
                "excluded_rows": 0,
                "positive_support": 41,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H2": {
                  "support": 41,
                  "tp": 24,
                  "recall": 0.5853658536585366,
                  "recall_wilson_95": [
                    0.4336710313403013,
                    0.7224345054574786
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "HR": {
              "n": 1444,
              "correct": 1063,
              "accuracy": 0.7361495844875346,
              "wilson_95": [
                0.7128131434529974,
                0.7582329064062927
              ],
              "macro_f1": 0.524805941091774,
              "errors": 0,
              "probability_rows": 1444,
              "multiclass_brier": 0.3402300802077562,
              "nll": 0.5105083345343254,
              "ece_10_equal_width": 0.04644799168975068,
              "p50_ms": 30.580042162910104,
              "p95_ms": 188.68350004777312,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1444,
              "per_class": {
                "safe": {
                  "support": 1368,
                  "negative_support": 76,
                  "tp": 1013,
                  "fp": 26,
                  "fn": 355,
                  "tn": 50,
                  "precision": 0.9749759384023099,
                  "recall": 0.7404970760233918,
                  "recall_wilson_95": [
                    0.7166170119992814,
                    0.7630302502766748
                  ],
                  "false_positive_rate": 0.34210526315789475,
                  "false_positive_wilson_95": [
                    0.24536326817494702,
                    0.4540410218456228
                  ]
                },
                "unsafe": {
                  "support": 76,
                  "negative_support": 1368,
                  "tp": 50,
                  "fp": 355,
                  "fn": 26,
                  "tn": 1013,
                  "precision": 0.12345679012345678,
                  "recall": 0.6578947368421053,
                  "recall_wilson_95": [
                    0.5459589781543774,
                    0.7546367318250531
                  ],
                  "false_positive_rate": 0.2595029239766082,
                  "false_positive_wilson_95": [
                    0.23696974972332507,
                    0.2833829880007185
                  ]
                }
              },
              "majority_baseline": 0.9473684210526315,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 50,
                "fp": 355,
                "fn": 26,
                "tn": 1013,
                "precision": 0.12345679012345678,
                "recall": 0.6578947368421053,
                "recall_wilson_95": [
                  0.5459589781543774,
                  0.7546367318250531
                ],
                "false_positive_rate": 0.2595029239766082,
                "false_positive_wilson_95": [
                  0.23696974972332507,
                  0.2833829880007185
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 50,
                "fp": 355,
                "fn": 26,
                "tn": 1013,
                "precision": 0.12345679012345678,
                "recall": 0.6578947368421053,
                "recall_wilson_95": [
                  0.5459589781543774,
                  0.7546367318250531
                ],
                "false_positive_rate": 0.2595029239766082,
                "false_positive_wilson_95": [
                  0.23696974972332507,
                  0.2833829880007185
                ],
                "f1": 0.2079002079002079,
                "negative_errors": 0
              },
              "harmful_f1": 0.2079002079002079,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.09969845324810675,
                "scored_rows": 1444,
                "excluded_rows": 0,
                "positive_support": 76,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "HR": {
                  "support": 76,
                  "tp": 50,
                  "recall": 0.6578947368421053,
                  "recall_wilson_95": [
                    0.5459589781543774,
                    0.7546367318250531
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S": {
              "n": 984,
              "correct": 502,
              "accuracy": 0.5101626016260162,
              "wilson_95": [
                0.47894961569279443,
                0.5412965481236913
              ],
              "macro_f1": 0.5017313252201977,
              "errors": 0,
              "probability_rows": 984,
              "multiclass_brier": 0.6117766757926829,
              "nll": 0.8472788450700258,
              "ece_10_equal_width": 0.16433750000000003,
              "p50_ms": 41.761665837839246,
              "p95_ms": 238.60166710801423,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 984,
              "per_class": {
                "safe": {
                  "support": 747,
                  "negative_support": 237,
                  "tp": 315,
                  "fp": 50,
                  "fn": 432,
                  "tn": 187,
                  "precision": 0.863013698630137,
                  "recall": 0.42168674698795183,
                  "recall_wilson_95": [
                    0.3867627158696995,
                    0.45741211141838684
                  ],
                  "false_positive_rate": 0.2109704641350211,
                  "false_positive_wilson_95": [
                    0.16384711356280787,
                    0.26731394703350675
                  ]
                },
                "unsafe": {
                  "support": 237,
                  "negative_support": 747,
                  "tp": 187,
                  "fp": 432,
                  "fn": 50,
                  "tn": 315,
                  "precision": 0.30210016155088854,
                  "recall": 0.7890295358649789,
                  "recall_wilson_95": [
                    0.7326860529664931,
                    0.8361528864371921
                  ],
                  "false_positive_rate": 0.5783132530120482,
                  "false_positive_wilson_95": [
                    0.5425878885816132,
                    0.6132372841303005
                  ]
                }
              },
              "majority_baseline": 0.7591463414634146,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 187,
                "fp": 432,
                "fn": 50,
                "tn": 315,
                "precision": 0.30210016155088854,
                "recall": 0.7890295358649789,
                "recall_wilson_95": [
                  0.7326860529664931,
                  0.8361528864371921
                ],
                "false_positive_rate": 0.5783132530120482,
                "false_positive_wilson_95": [
                  0.5425878885816132,
                  0.6132372841303005
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 187,
                "fp": 432,
                "fn": 50,
                "tn": 315,
                "precision": 0.30210016155088854,
                "recall": 0.7890295358649789,
                "recall_wilson_95": [
                  0.7326860529664931,
                  0.8361528864371921
                ],
                "false_positive_rate": 0.5783132530120482,
                "false_positive_wilson_95": [
                  0.5425878885816132,
                  0.6132372841303005
                ],
                "f1": 0.4369158878504673,
                "negative_errors": 0
              },
              "harmful_f1": 0.4369158878504673,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.2466380204145298,
                "scored_rows": 984,
                "excluded_rows": 0,
                "positive_support": 237,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S": {
                  "support": 237,
                  "tp": 187,
                  "recall": 0.7890295358649789,
                  "recall_wilson_95": [
                    0.7326860529664931,
                    0.8361528864371921
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S3": {
              "n": 994,
              "correct": 433,
              "accuracy": 0.43561368209255535,
              "wilson_95": [
                0.4050955989272478,
                0.4666275101212801
              ],
              "macro_f1": 0.37779825870008377,
              "errors": 0,
              "probability_rows": 994,
              "multiclass_brier": 0.6596211121529175,
              "nll": 0.8988873720302651,
              "ece_10_equal_width": 0.23071670020120721,
              "p50_ms": 41.83616698719561,
              "p95_ms": 227.56800008937716,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 994,
              "per_class": {
                "safe": {
                  "support": 909,
                  "negative_support": 85,
                  "tp": 368,
                  "fp": 20,
                  "fn": 541,
                  "tn": 65,
                  "precision": 0.9484536082474226,
                  "recall": 0.40484048404840484,
                  "recall_wilson_95": [
                    0.37339578012697144,
                    0.43708609673311344
                  ],
                  "false_positive_rate": 0.23529411764705882,
                  "false_positive_wilson_95": [
                    0.15779538725698744,
                    0.3356843394680986
                  ]
                },
                "unsafe": {
                  "support": 85,
                  "negative_support": 909,
                  "tp": 65,
                  "fp": 541,
                  "fn": 20,
                  "tn": 368,
                  "precision": 0.10726072607260725,
                  "recall": 0.7647058823529411,
                  "recall_wilson_95": [
                    0.6643156605319014,
                    0.8422046127430125
                  ],
                  "false_positive_rate": 0.5951595159515951,
                  "false_positive_wilson_95": [
                    0.5629139032668865,
                    0.6266042198730285
                  ]
                }
              },
              "majority_baseline": 0.9144869215291751,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 65,
                "fp": 541,
                "fn": 20,
                "tn": 368,
                "precision": 0.10726072607260725,
                "recall": 0.7647058823529411,
                "recall_wilson_95": [
                  0.6643156605319014,
                  0.8422046127430125
                ],
                "false_positive_rate": 0.5951595159515951,
                "false_positive_wilson_95": [
                  0.5629139032668865,
                  0.6266042198730285
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 65,
                "fp": 541,
                "fn": 20,
                "tn": 368,
                "precision": 0.10726072607260725,
                "recall": 0.7647058823529411,
                "recall_wilson_95": [
                  0.6643156605319014,
                  0.8422046127430125
                ],
                "false_positive_rate": 0.5951595159515951,
                "false_positive_wilson_95": [
                  0.5629139032668865,
                  0.6266042198730285
                ],
                "f1": 0.18813314037626627,
                "negative_errors": 0
              },
              "harmful_f1": 0.18813314037626627,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.09010088238245656,
                "scored_rows": 994,
                "excluded_rows": 0,
                "positive_support": 85,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S3": {
                  "support": 85,
                  "tp": 65,
                  "recall": 0.7647058823529411,
                  "recall_wilson_95": [
                    0.6643156605319014,
                    0.8422046127430125
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "SH": {
              "n": 1447,
              "correct": 980,
              "accuracy": 0.6772633033863165,
              "wilson_95": [
                0.6527323999140425,
                0.7008555106609614
              ],
              "macro_f1": 0.43095060652376205,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.3910917948617779,
              "nll": 0.5691611546614973,
              "ece_10_equal_width": 0.07788816529199571,
              "p50_ms": 30.76766710728407,
              "p95_ms": 191.23345892876387,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1396,
                  "negative_support": 51,
                  "tp": 966,
                  "fp": 37,
                  "fn": 430,
                  "tn": 14,
                  "precision": 0.963110667996012,
                  "recall": 0.6919770773638968,
                  "recall_wilson_95": [
                    0.6672595176863844,
                    0.7156409862403651
                  ],
                  "false_positive_rate": 0.7254901960784313,
                  "false_positive_wilson_95": [
                    0.5905334923922092,
                    0.8288572476535512
                  ]
                },
                "unsafe": {
                  "support": 51,
                  "negative_support": 1396,
                  "tp": 14,
                  "fp": 430,
                  "fn": 37,
                  "tn": 966,
                  "precision": 0.03153153153153153,
                  "recall": 0.27450980392156865,
                  "recall_wilson_95": [
                    0.17114275234644882,
                    0.4094665076077909
                  ],
                  "false_positive_rate": 0.30802292263610315,
                  "false_positive_wilson_95": [
                    0.2843590137596349,
                    0.33274048231361564
                  ]
                }
              },
              "majority_baseline": 0.9647546648237734,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 14,
                "fp": 430,
                "fn": 37,
                "tn": 966,
                "precision": 0.03153153153153153,
                "recall": 0.27450980392156865,
                "recall_wilson_95": [
                  0.17114275234644882,
                  0.4094665076077909
                ],
                "false_positive_rate": 0.30802292263610315,
                "false_positive_wilson_95": [
                  0.2843590137596349,
                  0.33274048231361564
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 14,
                "fp": 430,
                "fn": 37,
                "tn": 966,
                "precision": 0.03153153153153153,
                "recall": 0.27450980392156865,
                "recall_wilson_95": [
                  0.17114275234644882,
                  0.4094665076077909
                ],
                "false_positive_rate": 0.30802292263610315,
                "false_positive_wilson_95": [
                  0.2843590137596349,
                  0.33274048231361564
                ],
                "f1": 0.05656565656565657,
                "negative_errors": 0
              },
              "harmful_f1": 0.05656565656565657,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.03704884260243911,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 51,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "SH": {
                  "support": 51,
                  "tp": 14,
                  "recall": 0.27450980392156865,
                  "recall_wilson_95": [
                    0.17114275234644882,
                    0.4094665076077909
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V": {
              "n": 1450,
              "correct": 1007,
              "accuracy": 0.6944827586206896,
              "wilson_95": [
                0.6702856783116931,
                0.7176520824242658
              ],
              "macro_f1": 0.49810563885952347,
              "errors": 0,
              "probability_rows": 1450,
              "multiclass_brier": 0.38161977715862067,
              "nll": 0.5602404084477317,
              "ece_10_equal_width": 0.03454800000000003,
              "p50_ms": 30.704166973009706,
              "p95_ms": 194.69012483023107,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1450,
              "per_class": {
                "safe": {
                  "support": 1356,
                  "negative_support": 94,
                  "tp": 957,
                  "fp": 44,
                  "fn": 399,
                  "tn": 50,
                  "precision": 0.9560439560439561,
                  "recall": 0.7057522123893806,
                  "recall_wilson_95": [
                    0.6809433040331432,
                    0.7293986489794666
                  ],
                  "false_positive_rate": 0.46808510638297873,
                  "false_positive_wilson_95": [
                    0.3704589402235772,
                    0.5682173625184362
                  ]
                },
                "unsafe": {
                  "support": 94,
                  "negative_support": 1356,
                  "tp": 50,
                  "fp": 399,
                  "fn": 44,
                  "tn": 957,
                  "precision": 0.111358574610245,
                  "recall": 0.5319148936170213,
                  "recall_wilson_95": [
                    0.4317826374815637,
                    0.6295410597764227
                  ],
                  "false_positive_rate": 0.2942477876106195,
                  "false_positive_wilson_95": [
                    0.2706013510205334,
                    0.3190566959668568
                  ]
                }
              },
              "majority_baseline": 0.9351724137931035,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 50,
                "fp": 399,
                "fn": 44,
                "tn": 957,
                "precision": 0.111358574610245,
                "recall": 0.5319148936170213,
                "recall_wilson_95": [
                  0.4317826374815637,
                  0.6295410597764227
                ],
                "false_positive_rate": 0.2942477876106195,
                "false_positive_wilson_95": [
                  0.2706013510205334,
                  0.3190566959668568
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 50,
                "fp": 399,
                "fn": 44,
                "tn": 957,
                "precision": 0.111358574610245,
                "recall": 0.5319148936170213,
                "recall_wilson_95": [
                  0.4317826374815637,
                  0.6295410597764227
                ],
                "false_positive_rate": 0.2942477876106195,
                "false_positive_wilson_95": [
                  0.2706013510205334,
                  0.3190566959668568
                ],
                "f1": 0.1841620626151013,
                "negative_errors": 0
              },
              "harmful_f1": 0.1841620626151013,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.10624318674070451,
                "scored_rows": 1450,
                "excluded_rows": 0,
                "positive_support": 94,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V": {
                  "support": 94,
                  "tp": 50,
                  "recall": 0.5319148936170213,
                  "recall_wilson_95": [
                    0.4317826374815637,
                    0.6295410597764227
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V2": {
              "n": 1447,
              "correct": 1016,
              "accuracy": 0.7021423635107118,
              "wilson_95": [
                0.6780692603232075,
                0.7251450236658321
              ],
              "macro_f1": 0.43396173328383825,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.3896587864018123,
              "nll": 0.5746586703143105,
              "ece_10_equal_width": 0.01587206114219265,
              "p50_ms": 30.562750063836575,
              "p95_ms": 202.31037493795156,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1423,
                  "negative_support": 24,
                  "tp": 1006,
                  "fp": 14,
                  "fn": 417,
                  "tn": 10,
                  "precision": 0.9862745098039216,
                  "recall": 0.7069571328179901,
                  "recall_wilson_95": [
                    0.6827765035591971,
                    0.7300233883691025
                  ],
                  "false_positive_rate": 0.5833333333333334,
                  "false_positive_wilson_95": [
                    0.3883466586522876,
                    0.7553239739018824
                  ]
                },
                "unsafe": {
                  "support": 24,
                  "negative_support": 1423,
                  "tp": 10,
                  "fp": 417,
                  "fn": 14,
                  "tn": 1006,
                  "precision": 0.0234192037470726,
                  "recall": 0.4166666666666667,
                  "recall_wilson_95": [
                    0.24467602609811764,
                    0.6116533413477124
                  ],
                  "false_positive_rate": 0.29304286718200984,
                  "false_positive_wilson_95": [
                    0.26997661163089737,
                    0.31722349644080294
                  ]
                }
              },
              "majority_baseline": 0.9834139599170698,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 10,
                "fp": 417,
                "fn": 14,
                "tn": 1006,
                "precision": 0.0234192037470726,
                "recall": 0.4166666666666667,
                "recall_wilson_95": [
                  0.24467602609811764,
                  0.6116533413477124
                ],
                "false_positive_rate": 0.29304286718200984,
                "false_positive_wilson_95": [
                  0.26997661163089737,
                  0.31722349644080294
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 10,
                "fp": 417,
                "fn": 14,
                "tn": 1006,
                "precision": 0.0234192037470726,
                "recall": 0.4166666666666667,
                "recall_wilson_95": [
                  0.24467602609811764,
                  0.6116533413477124
                ],
                "false_positive_rate": 0.29304286718200984,
                "false_positive_wilson_95": [
                  0.26997661163089737,
                  0.31722349644080294
                ],
                "f1": 0.04434589800443459,
                "negative_errors": 0
              },
              "harmful_f1": 0.04434589800443459,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.03075164513147861,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 24,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V2": {
                  "support": 24,
                  "tp": 10,
                  "recall": 0.4166666666666667,
                  "recall_wilson_95": [
                    0.24467602609811764,
                    0.6116533413477124
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            }
          },
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 9298,
            "summary": {
              "n": 9298,
              "correct": 5848,
              "accuracy": 0.6289524628952463,
              "wilson_95": [
                0.6190818712644978,
                0.6387165453840723
              ],
              "macro_f1": 0.4933706593444915,
              "errors": 0,
              "probability_rows": 9298,
              "multiclass_brier": 0.455494301696372,
              "nll": 0.651483254799129,
              "ece_10_equal_width": 0.07492664050383899,
              "p50_ms": 31.325000105425715,
              "p95_ms": 202.31587486341596,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 9298,
              "per_class": {
                "safe": {
                  "support": 8528,
                  "negative_support": 770,
                  "tp": 5329,
                  "fp": 251,
                  "fn": 3199,
                  "tn": 519,
                  "precision": 0.9550179211469534,
                  "recall": 0.6248827392120075,
                  "recall_wilson_95": [
                    0.6145530682636524,
                    0.6350999533355579
                  ],
                  "false_positive_rate": 0.32597402597402597,
                  "false_positive_wilson_95": [
                    0.29380092801282226,
                    0.35987490319888077
                  ]
                },
                "unsafe": {
                  "support": 770,
                  "negative_support": 8528,
                  "tp": 519,
                  "fp": 3199,
                  "fn": 251,
                  "tn": 5329,
                  "precision": 0.1395911780527165,
                  "recall": 0.674025974025974,
                  "recall_wilson_95": [
                    0.6401250968011191,
                    0.7061990719871777
                  ],
                  "false_positive_rate": 0.3751172607879925,
                  "false_positive_wilson_95": [
                    0.36490004666444215,
                    0.38544693173634764
                  ]
                }
              },
              "majority_baseline": 0.9171864917186492,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 519,
                "fp": 3199,
                "fn": 251,
                "tn": 5329,
                "precision": 0.1395911780527165,
                "recall": 0.674025974025974,
                "recall_wilson_95": [
                  0.6401250968011191,
                  0.7061990719871777
                ],
                "false_positive_rate": 0.3751172607879925,
                "false_positive_wilson_95": [
                  0.36490004666444215,
                  0.38544693173634764
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 519,
                "fp": 3199,
                "fn": 251,
                "tn": 5329,
                "precision": 0.1395911780527165,
                "recall": 0.674025974025974,
                "recall_wilson_95": [
                  0.6401250968011191,
                  0.7061990719871777
                ],
                "false_positive_rate": 0.3751172607879925,
                "false_positive_wilson_95": [
                  0.36490004666444215,
                  0.38544693173634764
                ],
                "f1": 0.23128342245989306,
                "negative_errors": 0
              },
              "harmful_f1": 0.23128342245989306,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.11551582971319406,
                "scored_rows": 9298,
                "excluded_rows": 0,
                "positive_support": 770,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 119,
                  "recall": 0.7345679012345679,
                  "recall_wilson_95": [
                    0.6617113535920193,
                    0.7965576494693342
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 24,
                  "recall": 0.5853658536585366,
                  "recall_wilson_95": [
                    0.4336710313403013,
                    0.7224345054574786
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 50,
                  "recall": 0.6578947368421053,
                  "recall_wilson_95": [
                    0.5459589781543774,
                    0.7546367318250531
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 187,
                  "recall": 0.7890295358649789,
                  "recall_wilson_95": [
                    0.7326860529664931,
                    0.8361528864371921
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 65,
                  "recall": 0.7647058823529411,
                  "recall_wilson_95": [
                    0.6643156605319014,
                    0.8422046127430125
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 14,
                  "recall": 0.27450980392156865,
                  "recall_wilson_95": [
                    0.17114275234644882,
                    0.4094665076077909
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 50,
                  "recall": 0.5319148936170213,
                  "recall_wilson_95": [
                    0.4317826374815637,
                    0.6295410597764227
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 10,
                  "recall": 0.4166666666666667,
                  "recall_wilson_95": [
                    0.24467602609811764,
                    0.6116533413477124
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_categories_fine1000.json",
          "input_sha256": "1f71beffa073c5ad603c822f1c80c3643d85b3e2419f96c432ef36ff17ac453c",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 9298,
            "correct": 4893,
            "accuracy": 0.5262422026242203,
            "wilson_95": [
              0.5160844297841418,
              0.5363783005430067
            ],
            "macro_f1": 0.4504506565479123,
            "errors": 0,
            "probability_rows": 9298,
            "multiclass_brier": 0.6873714644467066,
            "nll": 1.0110209328857989,
            "ece_10_equal_width": 0.31313638275234634,
            "p50_ms": 41.15279112011194,
            "p95_ms": 97.07920788787305,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 9298,
            "per_class": {
              "safe": {
                "support": 8528,
                "negative_support": 770,
                "tp": 4173,
                "fp": 50,
                "fn": 4355,
                "tn": 720,
                "precision": 0.988160075775515,
                "recall": 0.4893292682926829,
                "recall_wilson_95": [
                  0.4787269443486175,
                  0.4999412012236565
                ],
                "false_positive_rate": 0.06493506493506493,
                "false_positive_wilson_95": [
                  0.049599656256584995,
                  0.08458992177267269
                ]
              },
              "unsafe": {
                "support": 770,
                "negative_support": 8528,
                "tp": 720,
                "fp": 4355,
                "fn": 50,
                "tn": 4173,
                "precision": 0.14187192118226602,
                "recall": 0.935064935064935,
                "recall_wilson_95": [
                  0.9154100782273273,
                  0.950400343743415
                ],
                "false_positive_rate": 0.510670731707317,
                "false_positive_wilson_95": [
                  0.5000587987763435,
                  0.5212730556513824
                ]
              }
            },
            "majority_baseline": 0.9171864917186492,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 720,
              "fp": 4355,
              "fn": 50,
              "tn": 4173,
              "precision": 0.14187192118226602,
              "recall": 0.935064935064935,
              "recall_wilson_95": [
                0.9154100782273273,
                0.950400343743415
              ],
              "false_positive_rate": 0.510670731707317,
              "false_positive_wilson_95": [
                0.5000587987763435,
                0.5212730556513824
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 720,
              "fp": 4355,
              "fn": 50,
              "tn": 4173,
              "precision": 0.14187192118226602,
              "recall": 0.935064935064935,
              "recall_wilson_95": [
                0.9154100782273273,
                0.950400343743415
              ],
              "false_positive_rate": 0.510670731707317,
              "false_positive_wilson_95": [
                0.5000587987763435,
                0.5212730556513824
              ],
              "f1": 0.24636441402908468,
              "negative_errors": 0
            },
            "harmful_f1": 0.24636441402908468,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.16592473464344268,
              "scored_rows": 9298,
              "excluded_rows": 0,
              "positive_support": 770,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 156,
                "recall": 0.9629629629629629,
                "recall_wilson_95": [
                  0.9215612866577387,
                  0.9829170088580903
                ]
              },
              "H2": {
                "support": 41,
                "tp": 41,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ]
              },
              "HR": {
                "support": 76,
                "tp": 71,
                "recall": 0.9342105263157895,
                "recall_wilson_95": [
                  0.8550647346629656,
                  0.9715734677804679
                ]
              },
              "S": {
                "support": 237,
                "tp": 232,
                "recall": 0.9789029535864979,
                "recall_wilson_95": [
                  0.9515730118442798,
                  0.990955741766206
                ]
              },
              "S3": {
                "support": 85,
                "tp": 84,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ]
              },
              "SH": {
                "support": 51,
                "tp": 35,
                "recall": 0.6862745098039216,
                "recall_wilson_95": [
                  0.5497300046930566,
                  0.7967232153447456
                ]
              },
              "V": {
                "support": 94,
                "tp": 85,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ]
              },
              "V2": {
                "support": 24,
                "tp": 16,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {
            "H": {
              "n": 771,
              "correct": 391,
              "accuracy": 0.5071335927367056,
              "wilson_95": [
                0.47189619826786144,
                0.5423002542770784
              ],
              "macro_f1": 0.5019041142468548,
              "errors": 0,
              "probability_rows": 771,
              "multiclass_brier": 0.7102628795590142,
              "nll": 1.042943628072592,
              "ece_10_equal_width": 0.32948936446173804,
              "p50_ms": 41.50329087860882,
              "p95_ms": 98.0157081503421,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 771,
              "per_class": {
                "safe": {
                  "support": 609,
                  "negative_support": 162,
                  "tp": 235,
                  "fp": 6,
                  "fn": 374,
                  "tn": 156,
                  "precision": 0.975103734439834,
                  "recall": 0.38587848932676516,
                  "recall_wilson_95": [
                    0.34804588725373803,
                    0.42514178143388454
                  ],
                  "false_positive_rate": 0.037037037037037035,
                  "false_positive_wilson_95": [
                    0.01708299114190962,
                    0.07843871334226116
                  ]
                },
                "unsafe": {
                  "support": 162,
                  "negative_support": 609,
                  "tp": 156,
                  "fp": 374,
                  "fn": 6,
                  "tn": 235,
                  "precision": 0.2943396226415094,
                  "recall": 0.9629629629629629,
                  "recall_wilson_95": [
                    0.9215612866577387,
                    0.9829170088580903
                  ],
                  "false_positive_rate": 0.6141215106732348,
                  "false_positive_wilson_95": [
                    0.5748582185661155,
                    0.651954112746262
                  ]
                }
              },
              "majority_baseline": 0.7898832684824902,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 156,
                "fp": 374,
                "fn": 6,
                "tn": 235,
                "precision": 0.2943396226415094,
                "recall": 0.9629629629629629,
                "recall_wilson_95": [
                  0.9215612866577387,
                  0.9829170088580903
                ],
                "false_positive_rate": 0.6141215106732348,
                "false_positive_wilson_95": [
                  0.5748582185661155,
                  0.651954112746262
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 156,
                "fp": 374,
                "fn": 6,
                "tn": 235,
                "precision": 0.2943396226415094,
                "recall": 0.9629629629629629,
                "recall_wilson_95": [
                  0.9215612866577387,
                  0.9829170088580903
                ],
                "false_positive_rate": 0.6141215106732348,
                "false_positive_wilson_95": [
                  0.5748582185661155,
                  0.651954112746262
                ],
                "f1": 0.4508670520231214,
                "negative_errors": 0
              },
              "harmful_f1": 0.4508670520231214,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.3856020906437183,
                "scored_rows": 771,
                "excluded_rows": 0,
                "positive_support": 162,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 156,
                  "recall": 0.9629629629629629,
                  "recall_wilson_95": [
                    0.9215612866577387,
                    0.9829170088580903
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "H2": {
              "n": 761,
              "correct": 282,
              "accuracy": 0.37056504599211565,
              "wilson_95": [
                0.3369818724648128,
                0.405448408048807
              ],
              "macro_f1": 0.323864216010877,
              "errors": 0,
              "probability_rows": 761,
              "multiclass_brier": 0.8986109980969859,
              "nll": 1.272602214718862,
              "ece_10_equal_width": 0.45283010624036146,
              "p50_ms": 41.213749907910824,
              "p95_ms": 107.26112499833107,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 761,
              "per_class": {
                "safe": {
                  "support": 720,
                  "negative_support": 41,
                  "tp": 241,
                  "fp": 0,
                  "fn": 479,
                  "tn": 41,
                  "precision": 1.0,
                  "recall": 0.3347222222222222,
                  "recall_wilson_95": [
                    0.30121099857898687,
                    0.3699877189029362
                  ],
                  "false_positive_rate": 0.0,
                  "false_positive_wilson_95": [
                    -6.938893903907228e-18,
                    0.08566757018443186
                  ]
                },
                "unsafe": {
                  "support": 41,
                  "negative_support": 720,
                  "tp": 41,
                  "fp": 479,
                  "fn": 0,
                  "tn": 241,
                  "precision": 0.07884615384615384,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ],
                  "false_positive_rate": 0.6652777777777777,
                  "false_positive_wilson_95": [
                    0.6300122810970638,
                    0.6987890014210131
                  ]
                }
              },
              "majority_baseline": 0.9461235216819974,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 41,
                "fp": 479,
                "fn": 0,
                "tn": 241,
                "precision": 0.07884615384615384,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ],
                "false_positive_rate": 0.6652777777777777,
                "false_positive_wilson_95": [
                  0.6300122810970638,
                  0.6987890014210131
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 41,
                "fp": 479,
                "fn": 0,
                "tn": 241,
                "precision": 0.07884615384615384,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ],
                "false_positive_rate": 0.6652777777777777,
                "false_positive_wilson_95": [
                  0.6300122810970638,
                  0.6987890014210131
                ],
                "f1": 0.14616755793226383,
                "negative_errors": 0
              },
              "harmful_f1": 0.14616755793226383,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.11465521758423486,
                "scored_rows": 761,
                "excluded_rows": 0,
                "positive_support": 41,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H2": {
                  "support": 41,
                  "tp": 41,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "HR": {
              "n": 1444,
              "correct": 816,
              "accuracy": 0.5650969529085873,
              "wilson_95": [
                0.539388096894763,
                0.590460374327446
              ],
              "macro_f1": 0.44395472327483654,
              "errors": 0,
              "probability_rows": 1444,
              "multiclass_brier": 0.6279853986842105,
              "nll": 0.9260909299362099,
              "ece_10_equal_width": 0.28206585872576184,
              "p50_ms": 40.5858748126775,
              "p95_ms": 82.7345410361886,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1444,
              "per_class": {
                "safe": {
                  "support": 1368,
                  "negative_support": 76,
                  "tp": 745,
                  "fp": 5,
                  "fn": 623,
                  "tn": 71,
                  "precision": 0.9933333333333333,
                  "recall": 0.5445906432748538,
                  "recall_wilson_95": [
                    0.5181123546098114,
                    0.5708192046571592
                  ],
                  "false_positive_rate": 0.06578947368421052,
                  "false_positive_wilson_95": [
                    0.028426532219532126,
                    0.14493526533703452
                  ]
                },
                "unsafe": {
                  "support": 76,
                  "negative_support": 1368,
                  "tp": 71,
                  "fp": 623,
                  "fn": 5,
                  "tn": 745,
                  "precision": 0.10230547550432277,
                  "recall": 0.9342105263157895,
                  "recall_wilson_95": [
                    0.8550647346629656,
                    0.9715734677804679
                  ],
                  "false_positive_rate": 0.4554093567251462,
                  "false_positive_wilson_95": [
                    0.42918079534284076,
                    0.4818876453901885
                  ]
                }
              },
              "majority_baseline": 0.9473684210526315,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 71,
                "fp": 623,
                "fn": 5,
                "tn": 745,
                "precision": 0.10230547550432277,
                "recall": 0.9342105263157895,
                "recall_wilson_95": [
                  0.8550647346629656,
                  0.9715734677804679
                ],
                "false_positive_rate": 0.4554093567251462,
                "false_positive_wilson_95": [
                  0.42918079534284076,
                  0.4818876453901885
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 71,
                "fp": 623,
                "fn": 5,
                "tn": 745,
                "precision": 0.10230547550432277,
                "recall": 0.9342105263157895,
                "recall_wilson_95": [
                  0.8550647346629656,
                  0.9715734677804679
                ],
                "false_positive_rate": 0.4554093567251462,
                "false_positive_wilson_95": [
                  0.42918079534284076,
                  0.4818876453901885
                ],
                "f1": 0.18441558441558442,
                "negative_errors": 0
              },
              "harmful_f1": 0.18441558441558442,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.14257767419625686,
                "scored_rows": 1444,
                "excluded_rows": 0,
                "positive_support": 76,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "HR": {
                  "support": 76,
                  "tp": 71,
                  "recall": 0.9342105263157895,
                  "recall_wilson_95": [
                    0.8550647346629656,
                    0.9715734677804679
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S": {
              "n": 984,
              "correct": 462,
              "accuracy": 0.4695121951219512,
              "wilson_95": [
                0.4385086639668486,
                0.5007528445836941
              ],
              "macro_f1": 0.46951000359410566,
              "errors": 0,
              "probability_rows": 984,
              "multiclass_brier": 0.8085465128007051,
              "nll": 1.2441673388850982,
              "ece_10_equal_width": 0.3567188441623317,
              "p50_ms": 42.789375176653266,
              "p95_ms": 110.49629095941782,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 984,
              "per_class": {
                "safe": {
                  "support": 747,
                  "negative_support": 237,
                  "tp": 230,
                  "fp": 5,
                  "fn": 517,
                  "tn": 232,
                  "precision": 0.9787234042553191,
                  "recall": 0.3078982597054886,
                  "recall_wilson_95": [
                    0.2758475708604701,
                    0.34191461231628834
                  ],
                  "false_positive_rate": 0.02109704641350211,
                  "false_positive_wilson_95": [
                    0.009044258233793949,
                    0.04842698815572015
                  ]
                },
                "unsafe": {
                  "support": 237,
                  "negative_support": 747,
                  "tp": 232,
                  "fp": 517,
                  "fn": 5,
                  "tn": 230,
                  "precision": 0.3097463284379172,
                  "recall": 0.9789029535864979,
                  "recall_wilson_95": [
                    0.9515730118442798,
                    0.990955741766206
                  ],
                  "false_positive_rate": 0.6921017402945113,
                  "false_positive_wilson_95": [
                    0.6580853876837116,
                    0.7241524291395298
                  ]
                }
              },
              "majority_baseline": 0.7591463414634146,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 232,
                "fp": 517,
                "fn": 5,
                "tn": 230,
                "precision": 0.3097463284379172,
                "recall": 0.9789029535864979,
                "recall_wilson_95": [
                  0.9515730118442798,
                  0.990955741766206
                ],
                "false_positive_rate": 0.6921017402945113,
                "false_positive_wilson_95": [
                  0.6580853876837116,
                  0.7241524291395298
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 232,
                "fp": 517,
                "fn": 5,
                "tn": 230,
                "precision": 0.3097463284379172,
                "recall": 0.9789029535864979,
                "recall_wilson_95": [
                  0.9515730118442798,
                  0.990955741766206
                ],
                "false_positive_rate": 0.6921017402945113,
                "false_positive_wilson_95": [
                  0.6580853876837116,
                  0.7241524291395298
                ],
                "f1": 0.47058823529411764,
                "negative_errors": 0
              },
              "harmful_f1": 0.47058823529411764,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.28271205680942774,
                "scored_rows": 984,
                "excluded_rows": 0,
                "positive_support": 237,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S": {
                  "support": 237,
                  "tp": 232,
                  "recall": 0.9789029535864979,
                  "recall_wilson_95": [
                    0.9515730118442798,
                    0.990955741766206
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S3": {
              "n": 994,
              "correct": 343,
              "accuracy": 0.34507042253521125,
              "wilson_95": [
                0.31616447155997696,
                0.37516925958804326
              ],
              "macro_f1": 0.32412098879164747,
              "errors": 0,
              "probability_rows": 994,
              "multiclass_brier": 0.945134341486698,
              "nll": 1.3792929675737768,
              "ece_10_equal_width": 0.45878796109322667,
              "p50_ms": 42.083708103746176,
              "p95_ms": 120.21066597662866,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 994,
              "per_class": {
                "safe": {
                  "support": 909,
                  "negative_support": 85,
                  "tp": 259,
                  "fp": 1,
                  "fn": 650,
                  "tn": 84,
                  "precision": 0.9961538461538462,
                  "recall": 0.28492849284928495,
                  "recall_wilson_95": [
                    0.2565380972931533,
                    0.3151290348010271
                  ],
                  "false_positive_rate": 0.011764705882352941,
                  "false_positive_wilson_95": [
                    0.002079787134664092,
                    0.06367170882493911
                  ]
                },
                "unsafe": {
                  "support": 85,
                  "negative_support": 909,
                  "tp": 84,
                  "fp": 650,
                  "fn": 1,
                  "tn": 259,
                  "precision": 0.11444141689373297,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ],
                  "false_positive_rate": 0.7150715071507151,
                  "false_positive_wilson_95": [
                    0.6848709651989728,
                    0.7434619027068468
                  ]
                }
              },
              "majority_baseline": 0.9144869215291751,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 84,
                "fp": 650,
                "fn": 1,
                "tn": 259,
                "precision": 0.11444141689373297,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ],
                "false_positive_rate": 0.7150715071507151,
                "false_positive_wilson_95": [
                  0.6848709651989728,
                  0.7434619027068468
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 84,
                "fp": 650,
                "fn": 1,
                "tn": 259,
                "precision": 0.11444141689373297,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ],
                "false_positive_rate": 0.7150715071507151,
                "false_positive_wilson_95": [
                  0.6848709651989728,
                  0.7434619027068468
                ],
                "f1": 0.20512820512820512,
                "negative_errors": 0
              },
              "harmful_f1": 0.20512820512820512,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.09818789376681021,
                "scored_rows": 994,
                "excluded_rows": 0,
                "positive_support": 85,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S3": {
                  "support": 85,
                  "tp": 84,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "SH": {
              "n": 1447,
              "correct": 858,
              "accuracy": 0.5929509329647547,
              "wilson_95": [
                0.5674239853051491,
                0.6179856598111785
              ],
              "macro_f1": 0.421343436092242,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.6150780318175535,
              "nll": 0.9319652451683109,
              "ece_10_equal_width": 0.2667564616447823,
              "p50_ms": 40.94654181972146,
              "p95_ms": 84.12929205223918,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1396,
                  "negative_support": 51,
                  "tp": 823,
                  "fp": 16,
                  "fn": 573,
                  "tn": 35,
                  "precision": 0.9809296781883194,
                  "recall": 0.589541547277937,
                  "recall_wilson_95": [
                    0.5635254823102956,
                    0.6150661706406136
                  ],
                  "false_positive_rate": 0.3137254901960784,
                  "false_positive_wilson_95": [
                    0.2032767846552545,
                    0.4502699953069435
                  ]
                },
                "unsafe": {
                  "support": 51,
                  "negative_support": 1396,
                  "tp": 35,
                  "fp": 573,
                  "fn": 16,
                  "tn": 823,
                  "precision": 0.05756578947368421,
                  "recall": 0.6862745098039216,
                  "recall_wilson_95": [
                    0.5497300046930566,
                    0.7967232153447456
                  ],
                  "false_positive_rate": 0.41045845272206305,
                  "false_positive_wilson_95": [
                    0.3849338293593864,
                    0.4364745176897044
                  ]
                }
              },
              "majority_baseline": 0.9647546648237734,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 35,
                "fp": 573,
                "fn": 16,
                "tn": 823,
                "precision": 0.05756578947368421,
                "recall": 0.6862745098039216,
                "recall_wilson_95": [
                  0.5497300046930566,
                  0.7967232153447456
                ],
                "false_positive_rate": 0.41045845272206305,
                "false_positive_wilson_95": [
                  0.3849338293593864,
                  0.4364745176897044
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 35,
                "fp": 573,
                "fn": 16,
                "tn": 823,
                "precision": 0.05756578947368421,
                "recall": 0.6862745098039216,
                "recall_wilson_95": [
                  0.5497300046930566,
                  0.7967232153447456
                ],
                "false_positive_rate": 0.41045845272206305,
                "false_positive_wilson_95": [
                  0.3849338293593864,
                  0.4364745176897044
                ],
                "f1": 0.1062215477996965,
                "negative_errors": 0
              },
              "harmful_f1": 0.1062215477996965,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.05664341423432107,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 51,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "SH": {
                  "support": 51,
                  "tp": 35,
                  "recall": 0.6862745098039216,
                  "recall_wilson_95": [
                    0.5497300046930566,
                    0.7967232153447456
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V": {
              "n": 1450,
              "correct": 868,
              "accuracy": 0.5986206896551725,
              "wilson_95": [
                0.5731621018504057,
                0.623558110153999
              ],
              "macro_f1": 0.4775570545584215,
              "errors": 0,
              "probability_rows": 1450,
              "multiclass_brier": 0.5789702874758621,
              "nll": 0.8472779662038589,
              "ece_10_equal_width": 0.2536809655172413,
              "p50_ms": 40.76000000350177,
              "p95_ms": 83.50754179991782,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1450,
              "per_class": {
                "safe": {
                  "support": 1356,
                  "negative_support": 94,
                  "tp": 783,
                  "fp": 9,
                  "fn": 573,
                  "tn": 85,
                  "precision": 0.9886363636363636,
                  "recall": 0.577433628318584,
                  "recall_wilson_95": [
                    0.5509595508090541,
                    0.603470216453756
                  ],
                  "false_positive_rate": 0.09574468085106383,
                  "false_positive_wilson_95": [
                    0.05119196076308952,
                    0.17204120730241473
                  ]
                },
                "unsafe": {
                  "support": 94,
                  "negative_support": 1356,
                  "tp": 85,
                  "fp": 573,
                  "fn": 9,
                  "tn": 783,
                  "precision": 0.12917933130699089,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ],
                  "false_positive_rate": 0.4225663716814159,
                  "false_positive_wilson_95": [
                    0.396529783546244,
                    0.4490404491909459
                  ]
                }
              },
              "majority_baseline": 0.9351724137931035,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 85,
                "fp": 573,
                "fn": 9,
                "tn": 783,
                "precision": 0.12917933130699089,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ],
                "false_positive_rate": 0.4225663716814159,
                "false_positive_wilson_95": [
                  0.396529783546244,
                  0.4490404491909459
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 85,
                "fp": 573,
                "fn": 9,
                "tn": 783,
                "precision": 0.12917933130699089,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ],
                "false_positive_rate": 0.4225663716814159,
                "false_positive_wilson_95": [
                  0.396529783546244,
                  0.4490404491909459
                ],
                "f1": 0.22606382978723405,
                "negative_errors": 0
              },
              "harmful_f1": 0.22606382978723405,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.2638569493326751,
                "scored_rows": 1450,
                "excluded_rows": 0,
                "positive_support": 94,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V": {
                  "support": 94,
                  "tp": 85,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V2": {
              "n": 1447,
              "correct": 873,
              "accuracy": 0.6033172080165861,
              "wilson_95": [
                0.577869298851217,
                0.6282180018542923
              ],
              "macro_f1": 0.4009655773269635,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.5447927975950242,
              "nll": 0.7728077930296039,
              "ece_10_equal_width": 0.23991319972356595,
              "p50_ms": 40.76787503436208,
              "p95_ms": 84.7510420717299,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1423,
                  "negative_support": 24,
                  "tp": 857,
                  "fp": 8,
                  "fn": 566,
                  "tn": 16,
                  "precision": 0.9907514450867052,
                  "recall": 0.6022487702037947,
                  "recall_wilson_95": [
                    0.5765766600965061,
                    0.6273703153723142
                  ],
                  "false_positive_rate": 0.3333333333333333,
                  "false_positive_wilson_95": [
                    0.1797219032729775,
                    0.5329368316186824
                  ]
                },
                "unsafe": {
                  "support": 24,
                  "negative_support": 1423,
                  "tp": 16,
                  "fp": 566,
                  "fn": 8,
                  "tn": 857,
                  "precision": 0.027491408934707903,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ],
                  "false_positive_rate": 0.3977512297962052,
                  "false_positive_wilson_95": [
                    0.37262968462768575,
                    0.4234233399034939
                  ]
                }
              },
              "majority_baseline": 0.9834139599170698,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 16,
                "fp": 566,
                "fn": 8,
                "tn": 857,
                "precision": 0.027491408934707903,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ],
                "false_positive_rate": 0.3977512297962052,
                "false_positive_wilson_95": [
                  0.37262968462768575,
                  0.4234233399034939
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 16,
                "fp": 566,
                "fn": 8,
                "tn": 857,
                "precision": 0.027491408934707903,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ],
                "false_positive_rate": 0.3977512297962052,
                "false_positive_wilson_95": [
                  0.37262968462768575,
                  0.4234233399034939
                ],
                "f1": 0.052805280528052806,
                "negative_errors": 0
              },
              "harmful_f1": 0.052805280528052806,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.03162552691385748,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 24,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            }
          },
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 9298,
            "summary": {
              "n": 9298,
              "correct": 4893,
              "accuracy": 0.5262422026242203,
              "wilson_95": [
                0.5160844297841418,
                0.5363783005430067
              ],
              "macro_f1": 0.4504506565479123,
              "errors": 0,
              "probability_rows": 9298,
              "multiclass_brier": 0.6873714644467066,
              "nll": 1.0110209328857989,
              "ece_10_equal_width": 0.31313638275234634,
              "p50_ms": 41.15279112011194,
              "p95_ms": 97.07920788787305,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 9298,
              "per_class": {
                "safe": {
                  "support": 8528,
                  "negative_support": 770,
                  "tp": 4173,
                  "fp": 50,
                  "fn": 4355,
                  "tn": 720,
                  "precision": 0.988160075775515,
                  "recall": 0.4893292682926829,
                  "recall_wilson_95": [
                    0.4787269443486175,
                    0.4999412012236565
                  ],
                  "false_positive_rate": 0.06493506493506493,
                  "false_positive_wilson_95": [
                    0.049599656256584995,
                    0.08458992177267269
                  ]
                },
                "unsafe": {
                  "support": 770,
                  "negative_support": 8528,
                  "tp": 720,
                  "fp": 4355,
                  "fn": 50,
                  "tn": 4173,
                  "precision": 0.14187192118226602,
                  "recall": 0.935064935064935,
                  "recall_wilson_95": [
                    0.9154100782273273,
                    0.950400343743415
                  ],
                  "false_positive_rate": 0.510670731707317,
                  "false_positive_wilson_95": [
                    0.5000587987763435,
                    0.5212730556513824
                  ]
                }
              },
              "majority_baseline": 0.9171864917186492,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 720,
                "fp": 4355,
                "fn": 50,
                "tn": 4173,
                "precision": 0.14187192118226602,
                "recall": 0.935064935064935,
                "recall_wilson_95": [
                  0.9154100782273273,
                  0.950400343743415
                ],
                "false_positive_rate": 0.510670731707317,
                "false_positive_wilson_95": [
                  0.5000587987763435,
                  0.5212730556513824
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 720,
                "fp": 4355,
                "fn": 50,
                "tn": 4173,
                "precision": 0.14187192118226602,
                "recall": 0.935064935064935,
                "recall_wilson_95": [
                  0.9154100782273273,
                  0.950400343743415
                ],
                "false_positive_rate": 0.510670731707317,
                "false_positive_wilson_95": [
                  0.5000587987763435,
                  0.5212730556513824
                ],
                "f1": 0.24636441402908468,
                "negative_errors": 0
              },
              "harmful_f1": 0.24636441402908468,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.16592473464344268,
                "scored_rows": 9298,
                "excluded_rows": 0,
                "positive_support": 770,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 156,
                  "recall": 0.9629629629629629,
                  "recall_wilson_95": [
                    0.9215612866577387,
                    0.9829170088580903
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 41,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 71,
                  "recall": 0.9342105263157895,
                  "recall_wilson_95": [
                    0.8550647346629656,
                    0.9715734677804679
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 232,
                  "recall": 0.9789029535864979,
                  "recall_wilson_95": [
                    0.9515730118442798,
                    0.990955741766206
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 84,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 35,
                  "recall": 0.6862745098039216,
                  "recall_wilson_95": [
                    0.5497300046930566,
                    0.7967232153447456
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 85,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/openai_categories_fine5000.json",
          "input_sha256": "b6cd2974c8996d4791cdaa07e21048d89786eb3ff0ef1e241ef87483f3a04e3a",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 9298,
            "correct": 5096,
            "accuracy": 0.5480748548074855,
            "wilson_95": [
              0.5379411242737671,
              0.5581688775960503
            ],
            "macro_f1": 0.4667013370482511,
            "errors": 0,
            "probability_rows": 9298,
            "multiclass_brier": 0.7134097289094429,
            "nll": 1.098610000277657,
            "ece_10_equal_width": 0.3298540761454076,
            "p50_ms": 41.50545783340931,
            "p95_ms": 97.15187503024936,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 9298,
            "per_class": {
              "safe": {
                "support": 8528,
                "negative_support": 770,
                "tp": 4364,
                "fp": 38,
                "fn": 4164,
                "tn": 732,
                "precision": 0.9913675601999091,
                "recall": 0.5117260787992496,
                "recall_wilson_95": [
                  0.5011141721418051,
                  0.5223274261304214
                ],
                "false_positive_rate": 0.04935064935064935,
                "false_positive_wilson_95": [
                  0.03616379147899992,
                  0.06701168188264908
                ]
              },
              "unsafe": {
                "support": 770,
                "negative_support": 8528,
                "tp": 732,
                "fp": 4164,
                "fn": 38,
                "tn": 4364,
                "precision": 0.14950980392156862,
                "recall": 0.9506493506493506,
                "recall_wilson_95": [
                  0.932988318117351,
                  0.963836208521
                ],
                "false_positive_rate": 0.48827392120075047,
                "false_positive_wilson_95": [
                  0.4776725738695786,
                  0.498885827858195
                ]
              }
            },
            "majority_baseline": 0.9171864917186492,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 732,
              "fp": 4164,
              "fn": 38,
              "tn": 4364,
              "precision": 0.14950980392156862,
              "recall": 0.9506493506493506,
              "recall_wilson_95": [
                0.932988318117351,
                0.963836208521
              ],
              "false_positive_rate": 0.48827392120075047,
              "false_positive_wilson_95": [
                0.4776725738695786,
                0.498885827858195
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 770,
              "negative_support": 8528,
              "tp": 732,
              "fp": 4164,
              "fn": 38,
              "tn": 4364,
              "precision": 0.14950980392156862,
              "recall": 0.9506493506493506,
              "recall_wilson_95": [
                0.932988318117351,
                0.963836208521
              ],
              "false_positive_rate": 0.48827392120075047,
              "false_positive_wilson_95": [
                0.4776725738695786,
                0.498885827858195
              ],
              "f1": 0.2583833392163784,
              "negative_errors": 0
            },
            "harmful_f1": 0.2583833392163784,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.16210261327710143,
              "scored_rows": 9298,
              "excluded_rows": 0,
              "positive_support": 770,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "H": {
                "support": 162,
                "tp": 157,
                "recall": 0.9691358024691358,
                "recall_wilson_95": [
                  0.9297919442305332,
                  0.9867460618921736
                ]
              },
              "H2": {
                "support": 41,
                "tp": 41,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ]
              },
              "HR": {
                "support": 76,
                "tp": 73,
                "recall": 0.9605263157894737,
                "recall_wilson_95": [
                  0.8902521225345447,
                  0.9864853649054605
                ]
              },
              "S": {
                "support": 237,
                "tp": 234,
                "recall": 0.9873417721518988,
                "recall_wilson_95": [
                  0.9634512971313959,
                  0.9956858926660589
                ]
              },
              "S3": {
                "support": 85,
                "tp": 84,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ]
              },
              "SH": {
                "support": 51,
                "tp": 42,
                "recall": 0.8235294117647058,
                "recall_wilson_95": [
                  0.6974585747569192,
                  0.9042759653087368
                ]
              },
              "V": {
                "support": 94,
                "tp": 85,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ]
              },
              "V2": {
                "support": 24,
                "tp": 16,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {
            "H": {
              "n": 771,
              "correct": 407,
              "accuracy": 0.5278858625162127,
              "wilson_95": [
                0.492596542286776,
                0.5628986812979893
              ],
              "macro_f1": 0.5209152736807604,
              "errors": 0,
              "probability_rows": 771,
              "multiclass_brier": 0.7193194522178988,
              "nll": 1.0913429561604495,
              "ece_10_equal_width": 0.3350561608300907,
              "p50_ms": 41.906249942258,
              "p95_ms": 98.03041722625494,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 771,
              "per_class": {
                "safe": {
                  "support": 609,
                  "negative_support": 162,
                  "tp": 250,
                  "fp": 5,
                  "fn": 359,
                  "tn": 157,
                  "precision": 0.9803921568627451,
                  "recall": 0.41050903119868637,
                  "recall_wilson_95": [
                    0.37211895040095194,
                    0.4500210200087666
                  ],
                  "false_positive_rate": 0.030864197530864196,
                  "false_positive_wilson_95": [
                    0.013253938107826307,
                    0.07020805576946676
                  ]
                },
                "unsafe": {
                  "support": 162,
                  "negative_support": 609,
                  "tp": 157,
                  "fp": 359,
                  "fn": 5,
                  "tn": 250,
                  "precision": 0.30426356589147285,
                  "recall": 0.9691358024691358,
                  "recall_wilson_95": [
                    0.9297919442305332,
                    0.9867460618921736
                  ],
                  "false_positive_rate": 0.5894909688013136,
                  "false_positive_wilson_95": [
                    0.5499789799912334,
                    0.6278810495990481
                  ]
                }
              },
              "majority_baseline": 0.7898832684824902,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 157,
                "fp": 359,
                "fn": 5,
                "tn": 250,
                "precision": 0.30426356589147285,
                "recall": 0.9691358024691358,
                "recall_wilson_95": [
                  0.9297919442305332,
                  0.9867460618921736
                ],
                "false_positive_rate": 0.5894909688013136,
                "false_positive_wilson_95": [
                  0.5499789799912334,
                  0.6278810495990481
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 162,
                "negative_support": 609,
                "tp": 157,
                "fp": 359,
                "fn": 5,
                "tn": 250,
                "precision": 0.30426356589147285,
                "recall": 0.9691358024691358,
                "recall_wilson_95": [
                  0.9297919442305332,
                  0.9867460618921736
                ],
                "false_positive_rate": 0.5894909688013136,
                "false_positive_wilson_95": [
                  0.5499789799912334,
                  0.6278810495990481
                ],
                "f1": 0.4631268436578171,
                "negative_errors": 0
              },
              "harmful_f1": 0.4631268436578171,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.3893421849138895,
                "scored_rows": 771,
                "excluded_rows": 0,
                "positive_support": 162,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 157,
                  "recall": 0.9691358024691358,
                  "recall_wilson_95": [
                    0.9297919442305332,
                    0.9867460618921736
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "H2": {
              "n": 761,
              "correct": 293,
              "accuracy": 0.38501971090670173,
              "wilson_95": [
                0.3511070348695084,
                0.4200873767542656
              ],
              "macro_f1": 0.33380471380471377,
              "errors": 0,
              "probability_rows": 761,
              "multiclass_brier": 0.9610588379500656,
              "nll": 1.4392800800208971,
              "ece_10_equal_width": 0.47722339027595273,
              "p50_ms": 41.70566704124212,
              "p95_ms": 106.82141711004078,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 761,
              "per_class": {
                "safe": {
                  "support": 720,
                  "negative_support": 41,
                  "tp": 252,
                  "fp": 0,
                  "fn": 468,
                  "tn": 41,
                  "precision": 1.0,
                  "recall": 0.35,
                  "recall_wilson_95": [
                    0.316039958989652,
                    0.38555215435528667
                  ],
                  "false_positive_rate": 0.0,
                  "false_positive_wilson_95": [
                    -6.938893903907228e-18,
                    0.08566757018443186
                  ]
                },
                "unsafe": {
                  "support": 41,
                  "negative_support": 720,
                  "tp": 41,
                  "fp": 468,
                  "fn": 0,
                  "tn": 252,
                  "precision": 0.08055009823182711,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ],
                  "false_positive_rate": 0.65,
                  "false_positive_wilson_95": [
                    0.6144478456447134,
                    0.683960041010348
                  ]
                }
              },
              "majority_baseline": 0.9461235216819974,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 41,
                "fp": 468,
                "fn": 0,
                "tn": 252,
                "precision": 0.08055009823182711,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ],
                "false_positive_rate": 0.65,
                "false_positive_wilson_95": [
                  0.6144478456447134,
                  0.683960041010348
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 41,
                "negative_support": 720,
                "tp": 41,
                "fp": 468,
                "fn": 0,
                "tn": 252,
                "precision": 0.08055009823182711,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.9143324298155681,
                  0.9999999999999999
                ],
                "false_positive_rate": 0.65,
                "false_positive_wilson_95": [
                  0.6144478456447134,
                  0.683960041010348
                ],
                "f1": 0.14909090909090908,
                "negative_errors": 0
              },
              "harmful_f1": 0.14909090909090908,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.07014894006959325,
                "scored_rows": 761,
                "excluded_rows": 0,
                "positive_support": 41,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H2": {
                  "support": 41,
                  "tp": 41,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "HR": {
              "n": 1444,
              "correct": 839,
              "accuracy": 0.5810249307479224,
              "wilson_95": [
                0.555394690411503,
                0.6062252152586931
              ],
              "macro_f1": 0.4556501485774388,
              "errors": 0,
              "probability_rows": 1444,
              "multiclass_brier": 0.6764815736842105,
              "nll": 1.052777706597237,
              "ece_10_equal_width": 0.3129224376731302,
              "p50_ms": 40.95145780593157,
              "p95_ms": 83.24145805090666,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1444,
              "per_class": {
                "safe": {
                  "support": 1368,
                  "negative_support": 76,
                  "tp": 766,
                  "fp": 3,
                  "fn": 602,
                  "tn": 73,
                  "precision": 0.9960988296488946,
                  "recall": 0.5599415204678363,
                  "recall_wilson_95": [
                    0.5335053975384726,
                    0.5860419444269634
                  ],
                  "false_positive_rate": 0.039473684210526314,
                  "false_positive_wilson_95": [
                    0.013514635094539595,
                    0.10974787746545532
                  ]
                },
                "unsafe": {
                  "support": 76,
                  "negative_support": 1368,
                  "tp": 73,
                  "fp": 602,
                  "fn": 3,
                  "tn": 766,
                  "precision": 0.10814814814814815,
                  "recall": 0.9605263157894737,
                  "recall_wilson_95": [
                    0.8902521225345447,
                    0.9864853649054605
                  ],
                  "false_positive_rate": 0.44005847953216376,
                  "false_positive_wilson_95": [
                    0.4139580555730366,
                    0.46649460246152735
                  ]
                }
              },
              "majority_baseline": 0.9473684210526315,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 73,
                "fp": 602,
                "fn": 3,
                "tn": 766,
                "precision": 0.10814814814814815,
                "recall": 0.9605263157894737,
                "recall_wilson_95": [
                  0.8902521225345447,
                  0.9864853649054605
                ],
                "false_positive_rate": 0.44005847953216376,
                "false_positive_wilson_95": [
                  0.4139580555730366,
                  0.46649460246152735
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 76,
                "negative_support": 1368,
                "tp": 73,
                "fp": 602,
                "fn": 3,
                "tn": 766,
                "precision": 0.10814814814814815,
                "recall": 0.9605263157894737,
                "recall_wilson_95": [
                  0.8902521225345447,
                  0.9864853649054605
                ],
                "false_positive_rate": 0.44005847953216376,
                "false_positive_wilson_95": [
                  0.4139580555730366,
                  0.46649460246152735
                ],
                "f1": 0.19440745672436752,
                "negative_errors": 0
              },
              "harmful_f1": 0.19440745672436752,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.15530838237601216,
                "scored_rows": 1444,
                "excluded_rows": 0,
                "positive_support": 76,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "HR": {
                  "support": 76,
                  "tp": 73,
                  "recall": 0.9605263157894737,
                  "recall_wilson_95": [
                    0.8902521225345447,
                    0.9864853649054605
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S": {
              "n": 984,
              "correct": 514,
              "accuracy": 0.5223577235772358,
              "wilson_95": [
                0.4911219638427723,
                0.5534195965534962
              ],
              "macro_f1": 0.521311611153663,
              "errors": 0,
              "probability_rows": 984,
              "multiclass_brier": 0.7772130312601626,
              "nll": 1.2349691321609142,
              "ece_10_equal_width": 0.34597845528455295,
              "p50_ms": 43.14908315427601,
              "p95_ms": 110.21404108032584,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 984,
              "per_class": {
                "safe": {
                  "support": 747,
                  "negative_support": 237,
                  "tp": 280,
                  "fp": 3,
                  "fn": 467,
                  "tn": 234,
                  "precision": 0.9893992932862191,
                  "recall": 0.3748326639892905,
                  "recall_wilson_95": [
                    0.3408420414295636,
                    0.41010405004797595
                  ],
                  "false_positive_rate": 0.012658227848101266,
                  "false_positive_wilson_95": [
                    0.004314107333941039,
                    0.03654870286860414
                  ]
                },
                "unsafe": {
                  "support": 237,
                  "negative_support": 747,
                  "tp": 234,
                  "fp": 467,
                  "fn": 3,
                  "tn": 280,
                  "precision": 0.3338088445078459,
                  "recall": 0.9873417721518988,
                  "recall_wilson_95": [
                    0.9634512971313959,
                    0.9956858926660589
                  ],
                  "false_positive_rate": 0.6251673360107095,
                  "false_positive_wilson_95": [
                    0.5898959499520241,
                    0.6591579585704365
                  ]
                }
              },
              "majority_baseline": 0.7591463414634146,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 234,
                "fp": 467,
                "fn": 3,
                "tn": 280,
                "precision": 0.3338088445078459,
                "recall": 0.9873417721518988,
                "recall_wilson_95": [
                  0.9634512971313959,
                  0.9956858926660589
                ],
                "false_positive_rate": 0.6251673360107095,
                "false_positive_wilson_95": [
                  0.5898959499520241,
                  0.6591579585704365
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 237,
                "negative_support": 747,
                "tp": 234,
                "fp": 467,
                "fn": 3,
                "tn": 280,
                "precision": 0.3338088445078459,
                "recall": 0.9873417721518988,
                "recall_wilson_95": [
                  0.9634512971313959,
                  0.9956858926660589
                ],
                "false_positive_rate": 0.6251673360107095,
                "false_positive_wilson_95": [
                  0.5898959499520241,
                  0.6591579585704365
                ],
                "f1": 0.4989339019189765,
                "negative_errors": 0
              },
              "harmful_f1": 0.4989339019189765,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.3071894137045301,
                "scored_rows": 984,
                "excluded_rows": 0,
                "positive_support": 237,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S": {
                  "support": 237,
                  "tp": 234,
                  "recall": 0.9873417721518988,
                  "recall_wilson_95": [
                    0.9634512971313959,
                    0.9956858926660589
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "S3": {
              "n": 994,
              "correct": 399,
              "accuracy": 0.4014084507042254,
              "wilson_95": [
                0.3713714744008734,
                0.43220453632968486
              ],
              "macro_f1": 0.3672346002621232,
              "errors": 0,
              "probability_rows": 994,
              "multiclass_brier": 0.9642056123138832,
              "nll": 1.482192869674799,
              "ece_10_equal_width": 0.4671617706237424,
              "p50_ms": 42.69024985842407,
              "p95_ms": 120.18816592171788,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 994,
              "per_class": {
                "safe": {
                  "support": 909,
                  "negative_support": 85,
                  "tp": 315,
                  "fp": 1,
                  "fn": 594,
                  "tn": 84,
                  "precision": 0.9968354430379747,
                  "recall": 0.3465346534653465,
                  "recall_wilson_95": [
                    0.3163038319647682,
                    0.378057113954097
                  ],
                  "false_positive_rate": 0.011764705882352941,
                  "false_positive_wilson_95": [
                    0.002079787134664092,
                    0.06367170882493911
                  ]
                },
                "unsafe": {
                  "support": 85,
                  "negative_support": 909,
                  "tp": 84,
                  "fp": 594,
                  "fn": 1,
                  "tn": 315,
                  "precision": 0.12389380530973451,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ],
                  "false_positive_rate": 0.6534653465346535,
                  "false_positive_wilson_95": [
                    0.621942886045903,
                    0.6836961680352317
                  ]
                }
              },
              "majority_baseline": 0.9144869215291751,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 84,
                "fp": 594,
                "fn": 1,
                "tn": 315,
                "precision": 0.12389380530973451,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ],
                "false_positive_rate": 0.6534653465346535,
                "false_positive_wilson_95": [
                  0.621942886045903,
                  0.6836961680352317
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 85,
                "negative_support": 909,
                "tp": 84,
                "fp": 594,
                "fn": 1,
                "tn": 315,
                "precision": 0.12389380530973451,
                "recall": 0.9882352941176471,
                "recall_wilson_95": [
                  0.9363282911750611,
                  0.997920212865336
                ],
                "false_positive_rate": 0.6534653465346535,
                "false_positive_wilson_95": [
                  0.621942886045903,
                  0.6836961680352317
                ],
                "f1": 0.22018348623853212,
                "negative_errors": 0
              },
              "harmful_f1": 0.22018348623853212,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.10147150259192665,
                "scored_rows": 994,
                "excluded_rows": 0,
                "positive_support": 85,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "S3": {
                  "support": 85,
                  "tp": 84,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "SH": {
              "n": 1447,
              "correct": 896,
              "accuracy": 0.6192121630960609,
              "wilson_95": [
                0.5939083735450612,
                0.6438846657305264
              ],
              "macro_f1": 0.44418511431091035,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.6080730588389772,
              "nll": 0.9542636034742451,
              "ece_10_equal_width": 0.27170110573600553,
              "p50_ms": 41.165582835674286,
              "p95_ms": 84.81345814652741,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1396,
                  "negative_support": 51,
                  "tp": 854,
                  "fp": 9,
                  "fn": 542,
                  "tn": 42,
                  "precision": 0.9895712630359212,
                  "recall": 0.6117478510028653,
                  "recall_wilson_95": [
                    0.5859092903443007,
                    0.6369730925384341
                  ],
                  "false_positive_rate": 0.17647058823529413,
                  "false_positive_wilson_95": [
                    0.09572403469126317,
                    0.3025414252430807
                  ]
                },
                "unsafe": {
                  "support": 51,
                  "negative_support": 1396,
                  "tp": 42,
                  "fp": 542,
                  "fn": 9,
                  "tn": 854,
                  "precision": 0.07191780821917808,
                  "recall": 0.8235294117647058,
                  "recall_wilson_95": [
                    0.6974585747569192,
                    0.9042759653087368
                  ],
                  "false_positive_rate": 0.38825214899713467,
                  "false_positive_wilson_95": [
                    0.36302690746156585,
                    0.4140907096556994
                  ]
                }
              },
              "majority_baseline": 0.9647546648237734,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 42,
                "fp": 542,
                "fn": 9,
                "tn": 854,
                "precision": 0.07191780821917808,
                "recall": 0.8235294117647058,
                "recall_wilson_95": [
                  0.6974585747569192,
                  0.9042759653087368
                ],
                "false_positive_rate": 0.38825214899713467,
                "false_positive_wilson_95": [
                  0.36302690746156585,
                  0.4140907096556994
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 51,
                "negative_support": 1396,
                "tp": 42,
                "fp": 542,
                "fn": 9,
                "tn": 854,
                "precision": 0.07191780821917808,
                "recall": 0.8235294117647058,
                "recall_wilson_95": [
                  0.6974585747569192,
                  0.9042759653087368
                ],
                "false_positive_rate": 0.38825214899713467,
                "false_positive_wilson_95": [
                  0.36302690746156585,
                  0.4140907096556994
                ],
                "f1": 0.13228346456692913,
                "negative_errors": 0
              },
              "harmful_f1": 0.13228346456692913,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.053819153774114316,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 51,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "SH": {
                  "support": 51,
                  "tp": 42,
                  "recall": 0.8235294117647058,
                  "recall_wilson_95": [
                    0.6974585747569192,
                    0.9042759653087368
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V": {
              "n": 1450,
              "correct": 881,
              "accuracy": 0.6075862068965517,
              "wilson_95": [
                0.5822007904566235,
                0.6324030771845452
              ],
              "macro_f1": 0.48336828474262966,
              "errors": 0,
              "probability_rows": 1450,
              "multiclass_brier": 0.6240484219172414,
              "nll": 0.9645662222456768,
              "ece_10_equal_width": 0.28217855172413797,
              "p50_ms": 41.05487512424588,
              "p95_ms": 82.42345810867846,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1450,
              "per_class": {
                "safe": {
                  "support": 1356,
                  "negative_support": 94,
                  "tp": 796,
                  "fp": 9,
                  "fn": 560,
                  "tn": 85,
                  "precision": 0.9888198757763975,
                  "recall": 0.5870206489675516,
                  "recall_wilson_95": [
                    0.5606042040799847,
                    0.6129454391296495
                  ],
                  "false_positive_rate": 0.09574468085106383,
                  "false_positive_wilson_95": [
                    0.05119196076308952,
                    0.17204120730241473
                  ]
                },
                "unsafe": {
                  "support": 94,
                  "negative_support": 1356,
                  "tp": 85,
                  "fp": 560,
                  "fn": 9,
                  "tn": 796,
                  "precision": 0.13178294573643412,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ],
                  "false_positive_rate": 0.41297935103244837,
                  "false_positive_wilson_95": [
                    0.3870545608703505,
                    0.4393957959200152
                  ]
                }
              },
              "majority_baseline": 0.9351724137931035,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 85,
                "fp": 560,
                "fn": 9,
                "tn": 796,
                "precision": 0.13178294573643412,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ],
                "false_positive_rate": 0.41297935103244837,
                "false_positive_wilson_95": [
                  0.3870545608703505,
                  0.4393957959200152
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 94,
                "negative_support": 1356,
                "tp": 85,
                "fp": 560,
                "fn": 9,
                "tn": 796,
                "precision": 0.13178294573643412,
                "recall": 0.9042553191489362,
                "recall_wilson_95": [
                  0.8279587926975853,
                  0.9488080392369105
                ],
                "false_positive_rate": 0.41297935103244837,
                "false_positive_wilson_95": [
                  0.3870545608703505,
                  0.4393957959200152
                ],
                "f1": 0.23004059539918809,
                "negative_errors": 0
              },
              "harmful_f1": 0.23004059539918809,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.2214579752400475,
                "scored_rows": 1450,
                "excluded_rows": 0,
                "positive_support": 94,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V": {
                  "support": 94,
                  "tp": 85,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "V2": {
              "n": 1447,
              "correct": 867,
              "accuracy": 0.5991706979958535,
              "wilson_95": [
                0.57368976912253,
                0.6241264693473068
              ],
              "macro_f1": 0.3990622834000676,
              "errors": 0,
              "probability_rows": 1447,
              "multiclass_brier": 0.5960838039668279,
              "nll": 0.891497870886719,
              "ece_10_equal_width": 0.2716401520387008,
              "p50_ms": 41.081749834120274,
              "p95_ms": 86.39183384366333,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 1447,
              "per_class": {
                "safe": {
                  "support": 1423,
                  "negative_support": 24,
                  "tp": 851,
                  "fp": 8,
                  "fn": 572,
                  "tn": 16,
                  "precision": 0.9906868451688009,
                  "recall": 0.5980323260716796,
                  "recall_wilson_95": [
                    0.5723269667459069,
                    0.6232098241674983
                  ],
                  "false_positive_rate": 0.3333333333333333,
                  "false_positive_wilson_95": [
                    0.1797219032729775,
                    0.5329368316186824
                  ]
                },
                "unsafe": {
                  "support": 24,
                  "negative_support": 1423,
                  "tp": 16,
                  "fp": 572,
                  "fn": 8,
                  "tn": 851,
                  "precision": 0.027210884353741496,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ],
                  "false_positive_rate": 0.40196767392832045,
                  "false_positive_wilson_95": [
                    0.3767901758325018,
                    0.42767303325409306
                  ]
                }
              },
              "majority_baseline": 0.9834139599170698,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 16,
                "fp": 572,
                "fn": 8,
                "tn": 851,
                "precision": 0.027210884353741496,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ],
                "false_positive_rate": 0.40196767392832045,
                "false_positive_wilson_95": [
                  0.3767901758325018,
                  0.42767303325409306
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 24,
                "negative_support": 1423,
                "tp": 16,
                "fp": 572,
                "fn": 8,
                "tn": 851,
                "precision": 0.027210884353741496,
                "recall": 0.6666666666666666,
                "recall_wilson_95": [
                  0.4670631683813174,
                  0.8202780967270225
                ],
                "false_positive_rate": 0.40196767392832045,
                "false_positive_wilson_95": [
                  0.3767901758325018,
                  0.42767303325409306
                ],
                "f1": 0.05228758169934641,
                "negative_errors": 0
              },
              "harmful_f1": 0.05228758169934641,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.03368137841910133,
                "scored_rows": 1447,
                "excluded_rows": 0,
                "positive_support": 24,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            }
          },
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 9298,
            "summary": {
              "n": 9298,
              "correct": 5096,
              "accuracy": 0.5480748548074855,
              "wilson_95": [
                0.5379411242737671,
                0.5581688775960503
              ],
              "macro_f1": 0.4667013370482511,
              "errors": 0,
              "probability_rows": 9298,
              "multiclass_brier": 0.7134097289094429,
              "nll": 1.098610000277657,
              "ece_10_equal_width": 0.3298540761454076,
              "p50_ms": 41.50545783340931,
              "p95_ms": 97.15187503024936,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 9298,
              "per_class": {
                "safe": {
                  "support": 8528,
                  "negative_support": 770,
                  "tp": 4364,
                  "fp": 38,
                  "fn": 4164,
                  "tn": 732,
                  "precision": 0.9913675601999091,
                  "recall": 0.5117260787992496,
                  "recall_wilson_95": [
                    0.5011141721418051,
                    0.5223274261304214
                  ],
                  "false_positive_rate": 0.04935064935064935,
                  "false_positive_wilson_95": [
                    0.03616379147899992,
                    0.06701168188264908
                  ]
                },
                "unsafe": {
                  "support": 770,
                  "negative_support": 8528,
                  "tp": 732,
                  "fp": 4164,
                  "fn": 38,
                  "tn": 4364,
                  "precision": 0.14950980392156862,
                  "recall": 0.9506493506493506,
                  "recall_wilson_95": [
                    0.932988318117351,
                    0.963836208521
                  ],
                  "false_positive_rate": 0.48827392120075047,
                  "false_positive_wilson_95": [
                    0.4776725738695786,
                    0.498885827858195
                  ]
                }
              },
              "majority_baseline": 0.9171864917186492,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 732,
                "fp": 4164,
                "fn": 38,
                "tn": 4364,
                "precision": 0.14950980392156862,
                "recall": 0.9506493506493506,
                "recall_wilson_95": [
                  0.932988318117351,
                  0.963836208521
                ],
                "false_positive_rate": 0.48827392120075047,
                "false_positive_wilson_95": [
                  0.4776725738695786,
                  0.498885827858195
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 770,
                "negative_support": 8528,
                "tp": 732,
                "fp": 4164,
                "fn": 38,
                "tn": 4364,
                "precision": 0.14950980392156862,
                "recall": 0.9506493506493506,
                "recall_wilson_95": [
                  0.932988318117351,
                  0.963836208521
                ],
                "false_positive_rate": 0.48827392120075047,
                "false_positive_wilson_95": [
                  0.4776725738695786,
                  0.498885827858195
                ],
                "f1": 0.2583833392163784,
                "negative_errors": 0
              },
              "harmful_f1": 0.2583833392163784,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.16210261327710143,
                "scored_rows": 9298,
                "excluded_rows": 0,
                "positive_support": 770,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "H": {
                  "support": 162,
                  "tp": 157,
                  "recall": 0.9691358024691358,
                  "recall_wilson_95": [
                    0.9297919442305332,
                    0.9867460618921736
                  ]
                },
                "H2": {
                  "support": 41,
                  "tp": 41,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.9143324298155681,
                    0.9999999999999999
                  ]
                },
                "HR": {
                  "support": 76,
                  "tp": 73,
                  "recall": 0.9605263157894737,
                  "recall_wilson_95": [
                    0.8902521225345447,
                    0.9864853649054605
                  ]
                },
                "S": {
                  "support": 237,
                  "tp": 234,
                  "recall": 0.9873417721518988,
                  "recall_wilson_95": [
                    0.9634512971313959,
                    0.9956858926660589
                  ]
                },
                "S3": {
                  "support": 85,
                  "tp": 84,
                  "recall": 0.9882352941176471,
                  "recall_wilson_95": [
                    0.9363282911750611,
                    0.997920212865336
                  ]
                },
                "SH": {
                  "support": 51,
                  "tp": 42,
                  "recall": 0.8235294117647058,
                  "recall_wilson_95": [
                    0.6974585747569192,
                    0.9042759653087368
                  ]
                },
                "V": {
                  "support": 94,
                  "tp": 85,
                  "recall": 0.9042553191489362,
                  "recall_wilson_95": [
                    0.8279587926975853,
                    0.9488080392369105
                  ]
                },
                "V2": {
                  "support": 24,
                  "tp": 16,
                  "recall": 0.6666666666666666,
                  "recall_wilson_95": [
                    0.4670631683813174,
                    0.8202780967270225
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": -0.10271026027102603,
          "harmful_f1_delta_fine_minus_reference": 0.015080991569191626,
          "accuracy_interval": [
            -0.117774203238849,
            -0.08753819707122011
          ],
          "harmful_f1_interval": [
            0.0035757635804659596,
            0.026813177045667845
          ],
          "confidence": 0.95,
          "n_rows": 9298,
          "n_text_groups": 1653,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": -0.08087760808776079,
          "harmful_f1_delta_fine_minus_reference": 0.027099916756485348,
          "accuracy_interval": [
            -0.09794372600127918,
            -0.06350530272364202
          ],
          "harmful_f1_interval": [
            0.015175349058581282,
            0.03953227292221839
          ],
          "confidence": 0.95,
          "n_rows": 9298,
          "n_text_groups": 1653,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.021832652183265244,
          "harmful_f1_delta_fine_minus_reference": 0.012018925187293722,
          "accuracy_interval": [
            0.011235955056179803,
            0.0321242949620028
          ],
          "harmful_f1_interval": [
            0.007254381235030026,
            0.016733871651212665
          ],
          "confidence": 0.95,
          "n_rows": 9298,
          "n_text_groups": 1653,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    },
    {
      "slug": "xstest",
      "n": 450,
      "runs": {
        "base": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/xstest_base.json",
          "input_sha256": "504f8d26a73a9db17d85407d92e97196399c26bc06e26814557906e29455f9aa",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-base-2048",
          "weights_identity": "sha256:891102d372688fc2a094dac56a384bc537b87c63f21f9f3dac0be2b7cbc8d86c",
          "config_sha256": "29f01d17cf4de63e40f5095ef9a314e278260a7585d6a7afef6c910895e96074",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 450,
            "correct": 311,
            "accuracy": 0.6911111111111111,
            "wilson_95": [
              0.6469546770547778,
              0.7320322944020423
            ],
            "macro_f1": 0.6845290631698399,
            "errors": 0,
            "probability_rows": 450,
            "multiclass_brier": 0.409911144,
            "nll": 0.5995723172602009,
            "ece_10_equal_width": 0.021201333333333326,
            "p50_ms": 28.651000000536442,
            "p95_ms": 173.00958302803338,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 450,
            "per_class": {
              "safe": {
                "support": 250,
                "negative_support": 200,
                "tp": 188,
                "fp": 77,
                "fn": 62,
                "tn": 123,
                "precision": 0.7094339622641509,
                "recall": 0.752,
                "recall_wilson_95": [
                  0.6949243835691702,
                  0.8014484334851256
                ],
                "false_positive_rate": 0.385,
                "false_positive_wilson_95": [
                  0.32033309726923215,
                  0.45400132779753916
                ]
              },
              "unsafe": {
                "support": 200,
                "negative_support": 250,
                "tp": 123,
                "fp": 62,
                "fn": 77,
                "tn": 188,
                "precision": 0.6648648648648648,
                "recall": 0.615,
                "recall_wilson_95": [
                  0.5459986722024608,
                  0.6796669027307678
                ],
                "false_positive_rate": 0.248,
                "false_positive_wilson_95": [
                  0.19855156651487427,
                  0.3050756164308297
                ]
              }
            },
            "majority_baseline": 0.5555555555555556,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 123,
              "fp": 62,
              "fn": 77,
              "tn": 188,
              "precision": 0.6648648648648648,
              "recall": 0.615,
              "recall_wilson_95": [
                0.5459986722024608,
                0.6796669027307678
              ],
              "false_positive_rate": 0.248,
              "false_positive_wilson_95": [
                0.19855156651487427,
                0.3050756164308297
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 123,
              "fp": 62,
              "fn": 77,
              "tn": 188,
              "precision": 0.6648648648648648,
              "recall": 0.615,
              "recall_wilson_95": [
                0.5459986722024608,
                0.6796669027307678
              ],
              "false_positive_rate": 0.248,
              "false_positive_wilson_95": [
                0.19855156651487427,
                0.3050756164308297
              ],
              "f1": 0.638961038961039,
              "negative_errors": 0
            },
            "harmful_f1": 0.638961038961039,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.7218969438836619,
              "scored_rows": 450,
              "excluded_rows": 0,
              "positive_support": 200,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "contrast_definitions": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              },
              "contrast_discr": {
                "support": 25,
                "tp": 1,
                "recall": 0.04,
                "recall_wilson_95": [
                  0.007096233501987265,
                  0.19544063736193734
                ]
              },
              "contrast_figurative_language": {
                "support": 25,
                "tp": 20,
                "recall": 0.8,
                "recall_wilson_95": [
                  0.6086904963045967,
                  0.9113941531319742
                ]
              },
              "contrast_historical_events": {
                "support": 25,
                "tp": 8,
                "recall": 0.32,
                "recall_wilson_95": [
                  0.17205190306907978,
                  0.5158973072689775
                ]
              },
              "contrast_homonyms": {
                "support": 25,
                "tp": 21,
                "recall": 0.84,
                "recall_wilson_95": [
                  0.6534637839282114,
                  0.9359654854332355
                ]
              },
              "contrast_privacy": {
                "support": 25,
                "tp": 5,
                "recall": 0.2,
                "recall_wilson_95": [
                  0.08860584686802572,
                  0.3913095036954033
                ]
              },
              "contrast_safe_contexts": {
                "support": 25,
                "tp": 23,
                "recall": 0.92,
                "recall_wilson_95": [
                  0.7503389104960468,
                  0.9777795987151524
                ]
              },
              "contrast_safe_targets": {
                "support": 25,
                "tp": 21,
                "recall": 0.84,
                "recall_wilson_95": [
                  0.6534637839282114,
                  0.9359654854332355
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 450,
            "summary": {
              "n": 450,
              "correct": 311,
              "accuracy": 0.6911111111111111,
              "wilson_95": [
                0.6469546770547778,
                0.7320322944020423
              ],
              "macro_f1": 0.6845290631698399,
              "errors": 0,
              "probability_rows": 450,
              "multiclass_brier": 0.409911144,
              "nll": 0.5995723172602009,
              "ece_10_equal_width": 0.021201333333333326,
              "p50_ms": 28.651000000536442,
              "p95_ms": 173.00958302803338,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 450,
              "per_class": {
                "safe": {
                  "support": 250,
                  "negative_support": 200,
                  "tp": 188,
                  "fp": 77,
                  "fn": 62,
                  "tn": 123,
                  "precision": 0.7094339622641509,
                  "recall": 0.752,
                  "recall_wilson_95": [
                    0.6949243835691702,
                    0.8014484334851256
                  ],
                  "false_positive_rate": 0.385,
                  "false_positive_wilson_95": [
                    0.32033309726923215,
                    0.45400132779753916
                  ]
                },
                "unsafe": {
                  "support": 200,
                  "negative_support": 250,
                  "tp": 123,
                  "fp": 62,
                  "fn": 77,
                  "tn": 188,
                  "precision": 0.6648648648648648,
                  "recall": 0.615,
                  "recall_wilson_95": [
                    0.5459986722024608,
                    0.6796669027307678
                  ],
                  "false_positive_rate": 0.248,
                  "false_positive_wilson_95": [
                    0.19855156651487427,
                    0.3050756164308297
                  ]
                }
              },
              "majority_baseline": 0.5555555555555556,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 123,
                "fp": 62,
                "fn": 77,
                "tn": 188,
                "precision": 0.6648648648648648,
                "recall": 0.615,
                "recall_wilson_95": [
                  0.5459986722024608,
                  0.6796669027307678
                ],
                "false_positive_rate": 0.248,
                "false_positive_wilson_95": [
                  0.19855156651487427,
                  0.3050756164308297
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 123,
                "fp": 62,
                "fn": 77,
                "tn": 188,
                "precision": 0.6648648648648648,
                "recall": 0.615,
                "recall_wilson_95": [
                  0.5459986722024608,
                  0.6796669027307678
                ],
                "false_positive_rate": 0.248,
                "false_positive_wilson_95": [
                  0.19855156651487427,
                  0.3050756164308297
                ],
                "f1": 0.638961038961039,
                "negative_errors": 0
              },
              "harmful_f1": 0.638961038961039,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.7218969438836619,
                "scored_rows": 450,
                "excluded_rows": 0,
                "positive_support": 200,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "contrast_definitions": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                },
                "contrast_discr": {
                  "support": 25,
                  "tp": 1,
                  "recall": 0.04,
                  "recall_wilson_95": [
                    0.007096233501987265,
                    0.19544063736193734
                  ]
                },
                "contrast_figurative_language": {
                  "support": 25,
                  "tp": 20,
                  "recall": 0.8,
                  "recall_wilson_95": [
                    0.6086904963045967,
                    0.9113941531319742
                  ]
                },
                "contrast_historical_events": {
                  "support": 25,
                  "tp": 8,
                  "recall": 0.32,
                  "recall_wilson_95": [
                    0.17205190306907978,
                    0.5158973072689775
                  ]
                },
                "contrast_homonyms": {
                  "support": 25,
                  "tp": 21,
                  "recall": 0.84,
                  "recall_wilson_95": [
                    0.6534637839282114,
                    0.9359654854332355
                  ]
                },
                "contrast_privacy": {
                  "support": 25,
                  "tp": 5,
                  "recall": 0.2,
                  "recall_wilson_95": [
                    0.08860584686802572,
                    0.3913095036954033
                  ]
                },
                "contrast_safe_contexts": {
                  "support": 25,
                  "tp": 23,
                  "recall": 0.92,
                  "recall_wilson_95": [
                    0.7503389104960468,
                    0.9777795987151524
                  ]
                },
                "contrast_safe_targets": {
                  "support": 25,
                  "tp": 21,
                  "recall": 0.84,
                  "recall_wilson_95": [
                    0.6534637839282114,
                    0.9359654854332355
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine1000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/xstest_fine1000.json",
          "input_sha256": "b3c0fb14d5a5ae5a1c213bb96986c257183f6557e80296b7d46d68462ff8e76f",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-1000-s7",
          "weights_identity": "sha256:0d86e28e5d44ac3de06b83396deee2c9f40991f9c3f3bf295eaf1b49fc162b6c",
          "config_sha256": "c851065d14ebc3dba7225847c3dceb6112039f846f0a50cda92369bb8b0a69e2",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 450,
            "correct": 255,
            "accuracy": 0.5666666666666667,
            "wilson_95": [
              0.5205087504413833,
              0.6116960070435539
            ],
            "macro_f1": 0.5521565385145377,
            "errors": 0,
            "probability_rows": 450,
            "multiclass_brier": 0.5932857008,
            "nll": 0.8718699908499585,
            "ece_10_equal_width": 0.2521595555555556,
            "p50_ms": 27.962583117187023,
            "p95_ms": 40.05741607397795,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 450,
            "per_class": {
              "safe": {
                "support": 250,
                "negative_support": 200,
                "tp": 87,
                "fp": 32,
                "fn": 163,
                "tn": 168,
                "precision": 0.7310924369747899,
                "recall": 0.348,
                "recall_wilson_95": [
                  0.29165747573175427,
                  0.40894304731486075
                ],
                "false_positive_rate": 0.16,
                "false_positive_wilson_95": [
                  0.1156741203158702,
                  0.21714070162067106
                ]
              },
              "unsafe": {
                "support": 200,
                "negative_support": 250,
                "tp": 168,
                "fp": 163,
                "fn": 32,
                "tn": 87,
                "precision": 0.5075528700906344,
                "recall": 0.84,
                "recall_wilson_95": [
                  0.7828592983793289,
                  0.8843258796841297
                ],
                "false_positive_rate": 0.652,
                "false_positive_wilson_95": [
                  0.5910569526851392,
                  0.7083425242682457
                ]
              }
            },
            "majority_baseline": 0.5555555555555556,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 168,
              "fp": 163,
              "fn": 32,
              "tn": 87,
              "precision": 0.5075528700906344,
              "recall": 0.84,
              "recall_wilson_95": [
                0.7828592983793289,
                0.8843258796841297
              ],
              "false_positive_rate": 0.652,
              "false_positive_wilson_95": [
                0.5910569526851392,
                0.7083425242682457
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 168,
              "fp": 163,
              "fn": 32,
              "tn": 87,
              "precision": 0.5075528700906344,
              "recall": 0.84,
              "recall_wilson_95": [
                0.7828592983793289,
                0.8843258796841297
              ],
              "false_positive_rate": 0.652,
              "false_positive_wilson_95": [
                0.5910569526851392,
                0.7083425242682457
              ],
              "f1": 0.632768361581921,
              "negative_errors": 0
            },
            "harmful_f1": 0.632768361581921,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.7394750332981319,
              "scored_rows": 450,
              "excluded_rows": 0,
              "positive_support": 200,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "contrast_definitions": {
                "support": 25,
                "tp": 25,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8668077490609515,
                  0.9999999999999999
                ]
              },
              "contrast_discr": {
                "support": 25,
                "tp": 13,
                "recall": 0.52,
                "recall_wilson_95": [
                  0.3349851695035678,
                  0.6996871404588704
                ]
              },
              "contrast_figurative_language": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              },
              "contrast_historical_events": {
                "support": 25,
                "tp": 20,
                "recall": 0.8,
                "recall_wilson_95": [
                  0.6086904963045967,
                  0.9113941531319742
                ]
              },
              "contrast_homonyms": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              },
              "contrast_privacy": {
                "support": 25,
                "tp": 13,
                "recall": 0.52,
                "recall_wilson_95": [
                  0.3349851695035678,
                  0.6996871404588704
                ]
              },
              "contrast_safe_contexts": {
                "support": 25,
                "tp": 25,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8668077490609515,
                  0.9999999999999999
                ]
              },
              "contrast_safe_targets": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 450,
            "summary": {
              "n": 450,
              "correct": 255,
              "accuracy": 0.5666666666666667,
              "wilson_95": [
                0.5205087504413833,
                0.6116960070435539
              ],
              "macro_f1": 0.5521565385145377,
              "errors": 0,
              "probability_rows": 450,
              "multiclass_brier": 0.5932857008,
              "nll": 0.8718699908499585,
              "ece_10_equal_width": 0.2521595555555556,
              "p50_ms": 27.962583117187023,
              "p95_ms": 40.05741607397795,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 450,
              "per_class": {
                "safe": {
                  "support": 250,
                  "negative_support": 200,
                  "tp": 87,
                  "fp": 32,
                  "fn": 163,
                  "tn": 168,
                  "precision": 0.7310924369747899,
                  "recall": 0.348,
                  "recall_wilson_95": [
                    0.29165747573175427,
                    0.40894304731486075
                  ],
                  "false_positive_rate": 0.16,
                  "false_positive_wilson_95": [
                    0.1156741203158702,
                    0.21714070162067106
                  ]
                },
                "unsafe": {
                  "support": 200,
                  "negative_support": 250,
                  "tp": 168,
                  "fp": 163,
                  "fn": 32,
                  "tn": 87,
                  "precision": 0.5075528700906344,
                  "recall": 0.84,
                  "recall_wilson_95": [
                    0.7828592983793289,
                    0.8843258796841297
                  ],
                  "false_positive_rate": 0.652,
                  "false_positive_wilson_95": [
                    0.5910569526851392,
                    0.7083425242682457
                  ]
                }
              },
              "majority_baseline": 0.5555555555555556,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 168,
                "fp": 163,
                "fn": 32,
                "tn": 87,
                "precision": 0.5075528700906344,
                "recall": 0.84,
                "recall_wilson_95": [
                  0.7828592983793289,
                  0.8843258796841297
                ],
                "false_positive_rate": 0.652,
                "false_positive_wilson_95": [
                  0.5910569526851392,
                  0.7083425242682457
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 168,
                "fp": 163,
                "fn": 32,
                "tn": 87,
                "precision": 0.5075528700906344,
                "recall": 0.84,
                "recall_wilson_95": [
                  0.7828592983793289,
                  0.8843258796841297
                ],
                "false_positive_rate": 0.652,
                "false_positive_wilson_95": [
                  0.5910569526851392,
                  0.7083425242682457
                ],
                "f1": 0.632768361581921,
                "negative_errors": 0
              },
              "harmful_f1": 0.632768361581921,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.7394750332981319,
                "scored_rows": 450,
                "excluded_rows": 0,
                "positive_support": 200,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "contrast_definitions": {
                  "support": 25,
                  "tp": 25,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8668077490609515,
                    0.9999999999999999
                  ]
                },
                "contrast_discr": {
                  "support": 25,
                  "tp": 13,
                  "recall": 0.52,
                  "recall_wilson_95": [
                    0.3349851695035678,
                    0.6996871404588704
                  ]
                },
                "contrast_figurative_language": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                },
                "contrast_historical_events": {
                  "support": 25,
                  "tp": 20,
                  "recall": 0.8,
                  "recall_wilson_95": [
                    0.6086904963045967,
                    0.9113941531319742
                  ]
                },
                "contrast_homonyms": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                },
                "contrast_privacy": {
                  "support": 25,
                  "tp": 13,
                  "recall": 0.52,
                  "recall_wilson_95": [
                    0.3349851695035678,
                    0.6996871404588704
                  ]
                },
                "contrast_safe_contexts": {
                  "support": 25,
                  "tp": 25,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8668077490609515,
                    0.9999999999999999
                  ]
                },
                "contrast_safe_targets": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        },
        "fine5000": {
          "input": "/Users/alexanderollman/jevtest/bench/results/moderation-study/xstest_fine5000.json",
          "input_sha256": "39133728f8b04d68d801c2f522f544bf83d7df27f28a139090dc298fb51639ff",
          "checkpoint": "/Users/alexanderollman/jevtest/models/laya-moderation-aegis-5000-s7",
          "weights_identity": "sha256:80211679fcb1526df36467de18ca2b3c8d0423ba7fbb8f01d3fc06f1aabd67bd",
          "config_sha256": "24c6e49cfa80b67f61bc9d6116f5649225853a1ff8887952b312931f2ffa723b",
          "pad_to_multiple": 128,
          "checkpoint_verification": "local weight/config bytes verified",
          "summary": {
            "n": 450,
            "correct": 315,
            "accuracy": 0.7,
            "wilson_95": [
              0.6561126727401614,
              0.7405015997146502
            ],
            "macro_f1": 0.6960965287470172,
            "errors": 0,
            "probability_rows": 450,
            "multiclass_brier": 0.4687977870990848,
            "nll": 0.72082105819594,
            "ece_10_equal_width": 0.1764277653320888,
            "p50_ms": 27.50200009904802,
            "p95_ms": 38.34987501613796,
            "cost_usd": null,
            "reported_cost_usd": 0,
            "cost_reported_rows": 0,
            "cost_unknown_rows": 450,
            "per_class": {
              "safe": {
                "support": 250,
                "negative_support": 200,
                "tp": 132,
                "fp": 17,
                "fn": 118,
                "tn": 183,
                "precision": 0.8859060402684564,
                "recall": 0.528,
                "recall_wilson_95": [
                  0.4661625890384914,
                  0.5889899461897637
                ],
                "false_positive_rate": 0.085,
                "false_positive_wilson_95": [
                  0.05374575017747561,
                  0.13189587071565567
                ]
              },
              "unsafe": {
                "support": 200,
                "negative_support": 250,
                "tp": 183,
                "fp": 118,
                "fn": 17,
                "tn": 132,
                "precision": 0.6079734219269103,
                "recall": 0.915,
                "recall_wilson_95": [
                  0.8681041292843443,
                  0.9462542498225244
                ],
                "false_positive_rate": 0.472,
                "false_positive_wilson_95": [
                  0.4110100538102362,
                  0.5338374109615085
                ]
              }
            },
            "majority_baseline": 0.5555555555555556,
            "safety_positive_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 183,
              "fp": 118,
              "fn": 17,
              "tn": 132,
              "precision": 0.6079734219269103,
              "recall": 0.915,
              "recall_wilson_95": [
                0.8681041292843443,
                0.9462542498225244
              ],
              "false_positive_rate": 0.472,
              "false_positive_wilson_95": [
                0.4110100538102362,
                0.5338374109615085
              ]
            },
            "harmful_class": {
              "label": "unsafe",
              "support": 200,
              "negative_support": 250,
              "tp": 183,
              "fp": 118,
              "fn": 17,
              "tn": 132,
              "precision": 0.6079734219269103,
              "recall": 0.915,
              "recall_wilson_95": [
                0.8681041292843443,
                0.9462542498225244
              ],
              "false_positive_rate": 0.472,
              "false_positive_wilson_95": [
                0.4110100538102362,
                0.5338374109615085
              ],
              "f1": 0.7305389221556886,
              "negative_errors": 0
            },
            "harmful_f1": 0.7305389221556886,
            "harmful_f1_zero_division": 0,
            "average_precision": {
              "value": 0.8156506494295432,
              "scored_rows": 450,
              "excluded_rows": 0,
              "positive_support": 200,
              "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
            },
            "per_category_positive_recall": {
              "contrast_definitions": {
                "support": 25,
                "tp": 25,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8668077490609515,
                  0.9999999999999999
                ]
              },
              "contrast_discr": {
                "support": 25,
                "tp": 20,
                "recall": 0.8,
                "recall_wilson_95": [
                  0.6086904963045967,
                  0.9113941531319742
                ]
              },
              "contrast_figurative_language": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              },
              "contrast_historical_events": {
                "support": 25,
                "tp": 18,
                "recall": 0.72,
                "recall_wilson_95": [
                  0.5242339480963842,
                  0.8571614614904344
                ]
              },
              "contrast_homonyms": {
                "support": 25,
                "tp": 24,
                "recall": 0.96,
                "recall_wilson_95": [
                  0.8045593626380625,
                  0.9929037664980127
                ]
              },
              "contrast_privacy": {
                "support": 25,
                "tp": 22,
                "recall": 0.88,
                "recall_wilson_95": [
                  0.7004420607907267,
                  0.9583318284955964
                ]
              },
              "contrast_safe_contexts": {
                "support": 25,
                "tp": 25,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8668077490609515,
                  0.9999999999999999
                ]
              },
              "contrast_safe_targets": {
                "support": 25,
                "tp": 25,
                "recall": 1.0,
                "recall_wilson_95": [
                  0.8668077490609515,
                  0.9999999999999999
                ]
              }
            },
            "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
          },
          "policy_categories": {},
          "untruncated_sensitivity": {
            "n_excluded": 0,
            "n_retained": 450,
            "summary": {
              "n": 450,
              "correct": 315,
              "accuracy": 0.7,
              "wilson_95": [
                0.6561126727401614,
                0.7405015997146502
              ],
              "macro_f1": 0.6960965287470172,
              "errors": 0,
              "probability_rows": 450,
              "multiclass_brier": 0.4687977870990848,
              "nll": 0.72082105819594,
              "ece_10_equal_width": 0.1764277653320888,
              "p50_ms": 27.50200009904802,
              "p95_ms": 38.34987501613796,
              "cost_usd": null,
              "reported_cost_usd": 0,
              "cost_reported_rows": 0,
              "cost_unknown_rows": 450,
              "per_class": {
                "safe": {
                  "support": 250,
                  "negative_support": 200,
                  "tp": 132,
                  "fp": 17,
                  "fn": 118,
                  "tn": 183,
                  "precision": 0.8859060402684564,
                  "recall": 0.528,
                  "recall_wilson_95": [
                    0.4661625890384914,
                    0.5889899461897637
                  ],
                  "false_positive_rate": 0.085,
                  "false_positive_wilson_95": [
                    0.05374575017747561,
                    0.13189587071565567
                  ]
                },
                "unsafe": {
                  "support": 200,
                  "negative_support": 250,
                  "tp": 183,
                  "fp": 118,
                  "fn": 17,
                  "tn": 132,
                  "precision": 0.6079734219269103,
                  "recall": 0.915,
                  "recall_wilson_95": [
                    0.8681041292843443,
                    0.9462542498225244
                  ],
                  "false_positive_rate": 0.472,
                  "false_positive_wilson_95": [
                    0.4110100538102362,
                    0.5338374109615085
                  ]
                }
              },
              "majority_baseline": 0.5555555555555556,
              "safety_positive_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 183,
                "fp": 118,
                "fn": 17,
                "tn": 132,
                "precision": 0.6079734219269103,
                "recall": 0.915,
                "recall_wilson_95": [
                  0.8681041292843443,
                  0.9462542498225244
                ],
                "false_positive_rate": 0.472,
                "false_positive_wilson_95": [
                  0.4110100538102362,
                  0.5338374109615085
                ]
              },
              "harmful_class": {
                "label": "unsafe",
                "support": 200,
                "negative_support": 250,
                "tp": 183,
                "fp": 118,
                "fn": 17,
                "tn": 132,
                "precision": 0.6079734219269103,
                "recall": 0.915,
                "recall_wilson_95": [
                  0.8681041292843443,
                  0.9462542498225244
                ],
                "false_positive_rate": 0.472,
                "false_positive_wilson_95": [
                  0.4110100538102362,
                  0.5338374109615085
                ],
                "f1": 0.7305389221556886,
                "negative_errors": 0
              },
              "harmful_f1": 0.7305389221556886,
              "harmful_f1_zero_division": 0,
              "average_precision": {
                "value": 0.8156506494295432,
                "scored_rows": 450,
                "excluded_rows": 0,
                "positive_support": 200,
                "method": "noninterpolated average precision; tied scores grouped; valid probability rows only"
              },
              "per_category_positive_recall": {
                "contrast_definitions": {
                  "support": 25,
                  "tp": 25,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8668077490609515,
                    0.9999999999999999
                  ]
                },
                "contrast_discr": {
                  "support": 25,
                  "tp": 20,
                  "recall": 0.8,
                  "recall_wilson_95": [
                    0.6086904963045967,
                    0.9113941531319742
                  ]
                },
                "contrast_figurative_language": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                },
                "contrast_historical_events": {
                  "support": 25,
                  "tp": 18,
                  "recall": 0.72,
                  "recall_wilson_95": [
                    0.5242339480963842,
                    0.8571614614904344
                  ]
                },
                "contrast_homonyms": {
                  "support": 25,
                  "tp": 24,
                  "recall": 0.96,
                  "recall_wilson_95": [
                    0.8045593626380625,
                    0.9929037664980127
                  ]
                },
                "contrast_privacy": {
                  "support": 25,
                  "tp": 22,
                  "recall": 0.88,
                  "recall_wilson_95": [
                    0.7004420607907267,
                    0.9583318284955964
                  ]
                },
                "contrast_safe_contexts": {
                  "support": 25,
                  "tp": 25,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8668077490609515,
                    0.9999999999999999
                  ]
                },
                "contrast_safe_targets": {
                  "support": 25,
                  "tp": 25,
                  "recall": 1.0,
                  "recall_wilson_95": [
                    0.8668077490609515,
                    0.9999999999999999
                  ]
                }
              },
              "category_metric_scope": "Positive unsafe rows only; categories do not define negatives or category false-positive rates."
            },
            "scope": "Descriptive scores on untruncated inputs only; no paired confidence interval computed for this subset."
          }
        }
      },
      "comparisons": {
        "fine1000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine1000",
          "accuracy_delta_fine_minus_reference": -0.12444444444444447,
          "harmful_f1_delta_fine_minus_reference": -0.0061926773791179945,
          "accuracy_interval": [
            -0.17999999999999994,
            -0.06888888888888889
          ],
          "harmful_f1_interval": [
            -0.05670837296876837,
            0.04369254737545155
          ],
          "confidence": 0.95,
          "n_rows": 450,
          "n_text_groups": 450,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_base": {
          "primary": false,
          "reference": "base",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.008888888888888835,
          "harmful_f1_delta_fine_minus_reference": 0.09157788319464966,
          "accuracy_interval": [
            -0.0444444444444444,
            0.06222222222222218
          ],
          "harmful_f1_interval": [
            0.039136341885250184,
            0.14521901575269364
          ],
          "confidence": 0.95,
          "n_rows": 450,
          "n_text_groups": 450,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        },
        "fine5000_vs_fine1000": {
          "primary": false,
          "reference": "fine1000",
          "fine": "fine5000",
          "accuracy_delta_fine_minus_reference": 0.1333333333333333,
          "harmful_f1_delta_fine_minus_reference": 0.09777056057376765,
          "accuracy_interval": [
            0.0888888888888889,
            0.17555555555555558
          ],
          "harmful_f1_interval": [
            0.06546153657573126,
            0.13054226994352897
          ],
          "confidence": 0.95,
          "n_rows": 450,
          "n_text_groups": 450,
          "resamples": 5000,
          "seed": 7,
          "harmful_f1_zero_division": 0,
          "method": "paired text-cluster percentile bootstrap; retain all rows per sampled cluster",
          "limitation": "Conditional on these fitted checkpoints and test set; not training-seed robustness. Confidence level alone does not establish a prespecified multiple-comparison family."
        }
      },
      "pending": [],
      "complete": true
    }
  ],
  "skipped": [],
  "complete": true,
  "pad_to_multiple": 128,
  "selection": "Fixed slug_target.json files only; no selection by score.",
  "primary_family": "Four harmful-F1 contrasts: Aegis prompt/response by 1000/5000 decisions versus base; 98.75% Bonferroni intervals. Accuracy and transfer comparisons are secondary.",
  "study_sha256": "863bf377337873b33452aeaf23442bdfa36759b82f31b3784dac9352eee54f0d",
  "unavailable_raw_sources": [],
  "preflight_sha256": "272ebfa57fc2b74ede73725e8e0b6ba41576cc779596b91ec5332bd10c981386"
}