{ "date": "2026-09-11", "organization": "CodeIntegrity", "hardware": "Apple M5 Pro, 24 GB RAM; PyTorch MPS; float32", "protocol": "PROTOCOL.md", "models": [ { "id": "codeintegrity-ai/promptguard", "name": "CodeIntegrity PromptGuard", "revision": "c66a9b35c82079773c3119f48caf19be18c6ca22", "total_parameters": 149606402, "latency_p50_ms": 9.879625002213288, "latency_p95_ms": 13.447981603530932, "suites": { "safeguard": { "n": 5140, "benign": 4068, "attacks": 1072, "tn": 3750, "fp": 318, "fn": 501, "tp": 571, "accuracy": 0.8406614785992218, "false_positive_rate": 0.07817109144542773, "recall": 0.5326492537313433, "precision": 0.6422947131608548, "f1": 0.5823559408465069, "balanced_accuracy": 0.7272390811429578, "roc_auc": 0.8271505442918152, "pr_auc": 0.6004033980978213, "balanced_accuracy_ci95": [ 0.7116953947078766, 0.7427506526181774 ] }, "notinject": { "n": 339, "benign": 339, "attacks": 0, "tn": 183, "fp": 156, "fn": 0, "tp": 0, "accuracy": 0.5398230088495575, "false_positive_rate": 0.46017699115044247, "recall": null, "precision": null, "f1": null }, "bipia": { "n": 575, "benign": 140, "attacks": 435, "tn": 138, "fp": 2, "fn": 388, "tp": 47, "accuracy": 0.3217391304347826, "false_positive_rate": 0.014285714285714285, "recall": 0.10804597701149425, "precision": 0.9591836734693877, "f1": 0.19421487603305784, "balanced_accuracy": 0.54688013136289, "roc_auc": 0.6465270935960591, "pr_auc": 0.8534431702327413, "balanced_accuracy_ci95": [ 0.5295155993431855, 0.5643873152709359 ] } }, "bipia_by_task": { "email": { "n": 181, "benign": 44, "attacks": 137, "tn": 44, "fp": 0, "fn": 136, "tp": 1, "accuracy": 0.24861878453038674, "false_positive_rate": 0.0, "recall": 0.0072992700729927005, "precision": 1.0, "f1": 0.014492753623188406, "balanced_accuracy": 0.5036496350364964, "roc_auc": 0.6199402786994027, "pr_auc": 0.8547992259634798, "balanced_accuracy_ci95": [ 0.5, 0.5145985401459854 ] }, "table": { "n": 196, "benign": 46, "attacks": 150, "tn": 46, "fp": 0, "fn": 150, "tp": 0, "accuracy": 0.23469387755102042, "false_positive_rate": 0.0, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.5, "roc_auc": 0.6152173913043478, "pr_auc": 0.8455959971774819, "balanced_accuracy_ci95": [ 0.5, 0.5 ] }, "code": { "n": 198, "benign": 50, "attacks": 148, "tn": 48, "fp": 2, "fn": 102, "tp": 46, "accuracy": 0.47474747474747475, "false_positive_rate": 0.04, "recall": 0.3108108108108108, "precision": 0.9583333333333334, "f1": 0.46938775510204084, "balanced_accuracy": 0.6354054054054054, "roc_auc": 0.729527027027027, "pr_auc": 0.8863128799602265, "balanced_accuracy_ci95": [ 0.5885135135135136, 0.6824324324324325 ] } }, "bipia_by_position": { "start": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 150, "tp": 0, "accuracy": 0.0, "false_positive_rate": null, "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "middle": { "n": 135, "benign": 0, "attacks": 135, "tn": 0, "fp": 0, "fn": 123, "tp": 12, "accuracy": 0.08888888888888889, "false_positive_rate": null, "recall": 0.08888888888888889, "precision": 1.0, "f1": 0.16326530612244897 }, "end": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 115, "tp": 35, "accuracy": 0.23333333333333334, "false_positive_rate": null, "recall": 0.23333333333333334, "precision": 1.0, "f1": 0.3783783783783784 } }, "safeguard_by_source": { "alpaca": { "n": 2412, "benign": 2410, "attacks": 2, "tn": 2340, "fp": 70, "fn": 2, "tp": 0, "accuracy": 0.9701492537313433, "false_positive_rate": 0.029045643153526972, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.4854771784232365, "roc_auc": 0.4941908713692946, "pr_auc": 0.0012035278545119705, "balanced_accuracy_ci95": [ 0.481945020746888, 0.48858921161825725 ] }, "dolly": { "n": 527, "benign": 527, "attacks": 0, "tn": 515, "fp": 12, "fn": 0, "tp": 0, "accuracy": 0.9772296015180265, "false_positive_rate": 0.022770398481973434, "recall": null, "precision": null, "f1": null }, "openassistant": { "n": 1068, "benign": 1065, "attacks": 3, "tn": 865, "fp": 200, "fn": 2, "tp": 1, "accuracy": 0.8108614232209738, "false_positive_rate": 0.18779342723004694, "recall": 0.3333333333333333, "precision": 0.004975124378109453, "f1": 0.00980392156862745, "balanced_accuracy": 0.5727699530516431, "roc_auc": 0.7154929577464788, "pr_auc": 0.03485518430669813, "balanced_accuracy_ci95": [ 0.3971830985915493, 0.9051643192488263 ] }, "synthetic": { "n": 1009, "benign": 48, "attacks": 961, "tn": 16, "fp": 32, "fn": 454, "tp": 507, "accuracy": 0.5183349851337958, "false_positive_rate": 0.6666666666666666, "recall": 0.5275754422476587, "precision": 0.9406307977736549, "f1": 0.676, "balanced_accuracy": 0.43045438779049605, "roc_auc": 0.39034859521331944, "pr_auc": 0.9324414761723565, "balanced_accuracy_ci95": [ 0.3627384668747832, 0.5023304717308359 ] }, "synthetic_clean": { "n": 124, "benign": 18, "attacks": 106, "tn": 14, "fp": 4, "fn": 43, "tp": 63, "accuracy": 0.6209677419354839, "false_positive_rate": 0.2222222222222222, "recall": 0.5943396226415094, "precision": 0.9402985074626866, "f1": 0.7283236994219653, "balanced_accuracy": 0.6860587002096437, "roc_auc": 0.7824947589098532, "pr_auc": 0.9522169684313184, "balanced_accuracy_ci95": [ 0.5697064989517819, 0.7882599580712788 ] } }, "notinject_by_category": { "Common Queries": { "n": 126, "benign": 126, "attacks": 0, "tn": 106, "fp": 20, "fn": 0, "tp": 0, "accuracy": 0.8412698412698413, "false_positive_rate": 0.15873015873015872, "recall": null, "precision": null, "f1": null }, "Multilingual": { "n": 84, "benign": 84, "attacks": 0, "tn": 2, "fp": 82, "fn": 0, "tp": 0, "accuracy": 0.023809523809523808, "false_positive_rate": 0.9761904761904762, "recall": null, "precision": null, "f1": null }, "Technique Queries": { "n": 87, "benign": 87, "attacks": 0, "tn": 38, "fp": 49, "fn": 0, "tp": 0, "accuracy": 0.4367816091954023, "false_positive_rate": 0.5632183908045977, "recall": null, "precision": null, "f1": null }, "Virtual Creation": { "n": 42, "benign": 42, "attacks": 0, "tn": 37, "fp": 5, "fn": 0, "tp": 0, "accuracy": 0.8809523809523809, "false_positive_rate": 0.11904761904761904, "recall": null, "precision": null, "f1": null } } }, { "id": "meta-llama/Llama-Prompt-Guard-2-86M", "name": "Meta Prompt Guard 2 \u00b7 86M", "revision": "a8ded8e697ce7c355e395a0df51f94adb4a2fd27", "total_parameters": 278810882, "latency_p50_ms": 13.643062498886138, "latency_p95_ms": 28.425018102279846, "suites": { "safeguard": { "n": 5140, "benign": 4068, "attacks": 1072, "tn": 4046, "fp": 22, "fn": 301, "tp": 771, "accuracy": 0.93715953307393, "false_positive_rate": 0.005408062930186824, "recall": 0.7192164179104478, "precision": 0.9722572509457755, "f1": 0.8268096514745308, "balanced_accuracy": 0.8569041774901305, "roc_auc": 0.9858687297289365, "pr_auc": 0.9564292629700835, "balanced_accuracy_ci95": [ 0.8440632613114369, 0.8708973454079162 ] }, "notinject": { "n": 339, "benign": 339, "attacks": 0, "tn": 317, "fp": 22, "fn": 0, "tp": 0, "accuracy": 0.9351032448377581, "false_positive_rate": 0.06489675516224189, "recall": null, "precision": null, "f1": null }, "bipia": { "n": 575, "benign": 140, "attacks": 435, "tn": 140, "fp": 0, "fn": 430, "tp": 5, "accuracy": 0.25217391304347825, "false_positive_rate": 0.0, "recall": 0.011494252873563218, "precision": 1.0, "f1": 0.022727272727272728, "balanced_accuracy": 0.5057471264367817, "roc_auc": 0.6688587848932677, "pr_auc": 0.8561837783231421, "balanced_accuracy_ci95": [ 0.5011494252873563, 0.5114942528735632 ] } }, "bipia_by_task": { "email": { "n": 181, "benign": 44, "attacks": 137, "tn": 44, "fp": 0, "fn": 135, "tp": 2, "accuracy": 0.2541436464088398, "false_positive_rate": 0.0, "recall": 0.014598540145985401, "precision": 1.0, "f1": 0.02877697841726619, "balanced_accuracy": 0.5072992700729927, "roc_auc": 0.6887856668878567, "pr_auc": 0.8740130027975042, "balanced_accuracy_ci95": [ 0.5, 0.5182481751824818 ] }, "table": { "n": 196, "benign": 46, "attacks": 150, "tn": 46, "fp": 0, "fn": 147, "tp": 3, "accuracy": 0.25, "false_positive_rate": 0.0, "recall": 0.02, "precision": 1.0, "f1": 0.0392156862745098, "balanced_accuracy": 0.51, "roc_auc": 0.6688405797101449, "pr_auc": 0.875950554771781, "balanced_accuracy_ci95": [ 0.5, 0.5233333333333333 ] }, "code": { "n": 198, "benign": 50, "attacks": 148, "tn": 50, "fp": 0, "fn": 148, "tp": 0, "accuracy": 0.25252525252525254, "false_positive_rate": 0.0, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.5, "roc_auc": 0.6888513513513513, "pr_auc": 0.831708740693243, "balanced_accuracy_ci95": [ 0.5, 0.5 ] } }, "bipia_by_position": { "start": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 149, "tp": 1, "accuracy": 0.006666666666666667, "false_positive_rate": null, "recall": 0.006666666666666667, "precision": 1.0, "f1": 0.013245033112582781 }, "middle": { "n": 135, "benign": 0, "attacks": 135, "tn": 0, "fp": 0, "fn": 133, "tp": 2, "accuracy": 0.014814814814814815, "false_positive_rate": null, "recall": 0.014814814814814815, "precision": 1.0, "f1": 0.029197080291970802 }, "end": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 148, "tp": 2, "accuracy": 0.013333333333333334, "false_positive_rate": null, "recall": 0.013333333333333334, "precision": 1.0, "f1": 0.02631578947368421 } }, "safeguard_by_source": { "alpaca": { "n": 2412, "benign": 2410, "attacks": 2, "tn": 2408, "fp": 2, "fn": 2, "tp": 0, "accuracy": 0.9983416252072969, "false_positive_rate": 0.0008298755186721991, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.4995850622406639, "roc_auc": 0.9885892116182573, "pr_auc": 0.0825, "balanced_accuracy_ci95": [ 0.49896265560165975, 0.5 ] }, "dolly": { "n": 527, "benign": 527, "attacks": 0, "tn": 527, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null }, "openassistant": { "n": 1068, "benign": 1065, "attacks": 3, "tn": 1062, "fp": 3, "fn": 3, "tp": 0, "accuracy": 0.9943820224719101, "false_positive_rate": 0.0028169014084507044, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.49859154929577465, "roc_auc": 0.7715179968701095, "pr_auc": 0.07641242937853107, "balanced_accuracy_ci95": [ 0.49671361502347416, 0.5 ] }, "synthetic": { "n": 1009, "benign": 48, "attacks": 961, "tn": 31, "fp": 17, "fn": 236, "tp": 725, "accuracy": 0.7492566897918731, "false_positive_rate": 0.3541666666666667, "recall": 0.7544224765868887, "precision": 0.977088948787062, "f1": 0.8514386376981797, "balanced_accuracy": 0.700127904960111, "roc_auc": 0.768578737426292, "pr_auc": 0.9863146977279695, "balanced_accuracy_ci95": [ 0.6256248916059659, 0.7714731291189733 ] }, "synthetic_clean": { "n": 124, "benign": 18, "attacks": 106, "tn": 18, "fp": 0, "fn": 60, "tp": 46, "accuracy": 0.5161290322580645, "false_positive_rate": 0.0, "recall": 0.4339622641509434, "precision": 1.0, "f1": 0.6052631578947368, "balanced_accuracy": 0.7169811320754718, "roc_auc": 0.9564989517819706, "pr_auc": 0.9916877199677043, "balanced_accuracy_ci95": [ 0.6698113207547169, 0.7688679245283019 ] } }, "notinject_by_category": { "Common Queries": { "n": 126, "benign": 126, "attacks": 0, "tn": 124, "fp": 2, "fn": 0, "tp": 0, "accuracy": 0.9841269841269841, "false_positive_rate": 0.015873015873015872, "recall": null, "precision": null, "f1": null }, "Multilingual": { "n": 84, "benign": 84, "attacks": 0, "tn": 84, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null }, "Technique Queries": { "n": 87, "benign": 87, "attacks": 0, "tn": 69, "fp": 18, "fn": 0, "tp": 0, "accuracy": 0.7931034482758621, "false_positive_rate": 0.20689655172413793, "recall": null, "precision": null, "f1": null }, "Virtual Creation": { "n": 42, "benign": 42, "attacks": 0, "tn": 40, "fp": 2, "fn": 0, "tp": 0, "accuracy": 0.9523809523809523, "false_positive_rate": 0.047619047619047616, "recall": null, "precision": null, "f1": null } } }, { "id": "meta-llama/Llama-Prompt-Guard-2-22M", "name": "Meta Prompt Guard 2 \u00b7 22M", "revision": "11614a155199674a0a95e6602d6ab0417b790ed0", "total_parameters": 70830722, "latency_p50_ms": 12.256812500709202, "latency_p95_ms": 16.235544394658064, "suites": { "safeguard": { "n": 5140, "benign": 4068, "attacks": 1072, "tn": 4028, "fp": 40, "fn": 330, "tp": 742, "accuracy": 0.9280155642023347, "false_positive_rate": 0.00983284169124877, "recall": 0.6921641791044776, "precision": 0.948849104859335, "f1": 0.8004314994606256, "balanced_accuracy": 0.8411656687066145, "roc_auc": 0.9656900783692157, "pr_auc": 0.916086745044843, "balanced_accuracy_ci95": [ 0.828032874436813, 0.8557745701800731 ] }, "notinject": { "n": 339, "benign": 339, "attacks": 0, "tn": 313, "fp": 26, "fn": 0, "tp": 0, "accuracy": 0.9233038348082596, "false_positive_rate": 0.07669616519174041, "recall": null, "precision": null, "f1": null }, "bipia": { "n": 575, "benign": 140, "attacks": 435, "tn": 140, "fp": 0, "fn": 415, "tp": 20, "accuracy": 0.2782608695652174, "false_positive_rate": 0.0, "recall": 0.04597701149425287, "precision": 1.0, "f1": 0.08791208791208792, "balanced_accuracy": 0.5229885057471264, "roc_auc": 0.7086371100164203, "pr_auc": 0.8899331875517233, "balanced_accuracy_ci95": [ 0.5137931034482759, 0.5344827586206896 ] } }, "bipia_by_task": { "email": { "n": 181, "benign": 44, "attacks": 137, "tn": 44, "fp": 0, "fn": 133, "tp": 4, "accuracy": 0.26519337016574585, "false_positive_rate": 0.0, "recall": 0.029197080291970802, "precision": 1.0, "f1": 0.05673758865248227, "balanced_accuracy": 0.5145985401459854, "roc_auc": 0.7344061048440611, "pr_auc": 0.9067360240755077, "balanced_accuracy_ci95": [ 0.5036496350364964, 0.5328467153284672 ] }, "table": { "n": 196, "benign": 46, "attacks": 150, "tn": 46, "fp": 0, "fn": 134, "tp": 16, "accuracy": 0.3163265306122449, "false_positive_rate": 0.0, "recall": 0.10666666666666667, "precision": 1.0, "f1": 0.1927710843373494, "balanced_accuracy": 0.5533333333333333, "roc_auc": 0.7523188405797102, "pr_auc": 0.9229946438087226, "balanced_accuracy_ci95": [ 0.53, 0.58 ] }, "code": { "n": 198, "benign": 50, "attacks": 148, "tn": 50, "fp": 0, "fn": 148, "tp": 0, "accuracy": 0.25252525252525254, "false_positive_rate": 0.0, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.5, "roc_auc": 0.6508108108108108, "pr_auc": 0.8110881102696055, "balanced_accuracy_ci95": [ 0.5, 0.5 ] } }, "bipia_by_position": { "start": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 148, "tp": 2, "accuracy": 0.013333333333333334, "false_positive_rate": null, "recall": 0.013333333333333334, "precision": 1.0, "f1": 0.02631578947368421 }, "middle": { "n": 135, "benign": 0, "attacks": 135, "tn": 0, "fp": 0, "fn": 125, "tp": 10, "accuracy": 0.07407407407407407, "false_positive_rate": null, "recall": 0.07407407407407407, "precision": 1.0, "f1": 0.13793103448275862 }, "end": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 142, "tp": 8, "accuracy": 0.05333333333333334, "false_positive_rate": null, "recall": 0.05333333333333334, "precision": 1.0, "f1": 0.10126582278481013 } }, "safeguard_by_source": { "alpaca": { "n": 2412, "benign": 2410, "attacks": 2, "tn": 2404, "fp": 6, "fn": 2, "tp": 0, "accuracy": 0.9966832504145937, "false_positive_rate": 0.0024896265560165973, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.4987551867219917, "roc_auc": 0.962655601659751, "pr_auc": 0.018357487922705314, "balanced_accuracy_ci95": [ 0.4975103734439834, 0.4995850622406639 ] }, "dolly": { "n": 527, "benign": 527, "attacks": 0, "tn": 527, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null }, "openassistant": { "n": 1068, "benign": 1065, "attacks": 3, "tn": 1052, "fp": 13, "fn": 2, "tp": 1, "accuracy": 0.9859550561797753, "false_positive_rate": 0.012206572769953052, "recall": 0.3333333333333333, "precision": 0.07142857142857142, "f1": 0.11764705882352941, "balanced_accuracy": 0.6605633802816901, "roc_auc": 0.6751173708920188, "pr_auc": 0.03453907892872135, "balanced_accuracy_ci95": [ 0.4915492957746479, 0.9938967136150234 ] }, "synthetic": { "n": 1009, "benign": 48, "attacks": 961, "tn": 27, "fp": 21, "fn": 259, "tp": 702, "accuracy": 0.7224975222993062, "false_positive_rate": 0.4375, "recall": 0.7304890738813735, "precision": 0.970954356846473, "f1": 0.833729216152019, "balanced_accuracy": 0.6464945369406867, "roc_auc": 0.7198231009365245, "pr_auc": 0.9822176121859206, "balanced_accuracy_ci95": [ 0.5715070022545959, 0.719942334373916 ] }, "synthetic_clean": { "n": 124, "benign": 18, "attacks": 106, "tn": 18, "fp": 0, "fn": 67, "tp": 39, "accuracy": 0.4596774193548387, "false_positive_rate": 0.0, "recall": 0.36792452830188677, "precision": 1.0, "f1": 0.5379310344827586, "balanced_accuracy": 0.6839622641509434, "roc_auc": 0.970649895178197, "pr_auc": 0.9949699834873446, "balanced_accuracy_ci95": [ 0.6415094339622641, 0.7311320754716981 ] } }, "notinject_by_category": { "Common Queries": { "n": 126, "benign": 126, "attacks": 0, "tn": 119, "fp": 7, "fn": 0, "tp": 0, "accuracy": 0.9444444444444444, "false_positive_rate": 0.05555555555555555, "recall": null, "precision": null, "f1": null }, "Multilingual": { "n": 84, "benign": 84, "attacks": 0, "tn": 84, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null }, "Technique Queries": { "n": 87, "benign": 87, "attacks": 0, "tn": 68, "fp": 19, "fn": 0, "tp": 0, "accuracy": 0.7816091954022989, "false_positive_rate": 0.21839080459770116, "recall": null, "precision": null, "f1": null }, "Virtual Creation": { "n": 42, "benign": 42, "attacks": 0, "tn": 42, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null } } }, { "id": "protectai/deberta-v3-base-prompt-injection-v2", "name": "Protect AI DeBERTa v2", "revision": "90c9989b1a342275dd0d1a95aad283c04e075671", "total_parameters": 184423682, "latency_p50_ms": 13.44224949934869, "latency_p95_ms": 16.18554230153677, "suites": { "safeguard": { "n": 5140, "benign": 4068, "attacks": 1072, "tn": 4022, "fp": 46, "fn": 195, "tp": 877, "accuracy": 0.953112840466926, "false_positive_rate": 0.011307767944936086, "recall": 0.8180970149253731, "precision": 0.9501625135427952, "f1": 0.8791979949874686, "balanced_accuracy": 0.9033946234902186, "roc_auc": 0.9757352617443755, "pr_auc": 0.949824178180443, "balanced_accuracy_ci95": [ 0.8922466506882989, 0.9149346028889476 ] }, "notinject": { "n": 339, "benign": 339, "attacks": 0, "tn": 194, "fp": 145, "fn": 0, "tp": 0, "accuracy": 0.5722713864306784, "false_positive_rate": 0.4277286135693215, "recall": null, "precision": null, "f1": null }, "bipia": { "n": 575, "benign": 140, "attacks": 435, "tn": 111, "fp": 29, "fn": 352, "tp": 83, "accuracy": 0.3373913043478261, "false_positive_rate": 0.20714285714285716, "recall": 0.19080459770114944, "precision": 0.7410714285714286, "f1": 0.30347349177330896, "balanced_accuracy": 0.4918308702791461, "roc_auc": 0.4474466338259442, "pr_auc": 0.7331654422436329, "balanced_accuracy_ci95": [ 0.45085899014778325, 0.5296100164203612 ] } }, "bipia_by_task": { "email": { "n": 181, "benign": 44, "attacks": 137, "tn": 15, "fp": 29, "fn": 55, "tp": 82, "accuracy": 0.5359116022099447, "false_positive_rate": 0.6590909090909091, "recall": 0.5985401459854015, "precision": 0.7387387387387387, "f1": 0.6612903225806451, "balanced_accuracy": 0.4697246184472462, "roc_auc": 0.45968812209688126, "pr_auc": 0.736781674782027, "balanced_accuracy_ci95": [ 0.3873486230922362, 0.5557502488387523 ] }, "table": { "n": 196, "benign": 46, "attacks": 150, "tn": 46, "fp": 0, "fn": 149, "tp": 1, "accuracy": 0.23979591836734693, "false_positive_rate": 0.0, "recall": 0.006666666666666667, "precision": 1.0, "f1": 0.013245033112582781, "balanced_accuracy": 0.5033333333333333, "roc_auc": 0.29413043478260864, "pr_auc": 0.6744557338049569, "balanced_accuracy_ci95": [ 0.5, 0.5133333333333333 ] }, "code": { "n": 198, "benign": 50, "attacks": 148, "tn": 50, "fp": 0, "fn": 148, "tp": 0, "accuracy": 0.25252525252525254, "false_positive_rate": 0.0, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.5, "roc_auc": 0.45918918918918916, "pr_auc": 0.7409093590384619, "balanced_accuracy_ci95": [ 0.5, 0.5 ] } }, "bipia_by_position": { "start": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 123, "tp": 27, "accuracy": 0.18, "false_positive_rate": null, "recall": 0.18, "precision": 1.0, "f1": 0.3050847457627119 }, "middle": { "n": 135, "benign": 0, "attacks": 135, "tn": 0, "fp": 0, "fn": 110, "tp": 25, "accuracy": 0.18518518518518517, "false_positive_rate": null, "recall": 0.18518518518518517, "precision": 1.0, "f1": 0.3125 }, "end": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 119, "tp": 31, "accuracy": 0.20666666666666667, "false_positive_rate": null, "recall": 0.20666666666666667, "precision": 1.0, "f1": 0.3425414364640884 } }, "safeguard_by_source": { "alpaca": { "n": 2412, "benign": 2410, "attacks": 2, "tn": 2410, "fp": 0, "fn": 2, "tp": 0, "accuracy": 0.9991708126036484, "false_positive_rate": 0.0, "recall": 0.0, "precision": 0.0, "f1": 0.0, "balanced_accuracy": 0.5, "roc_auc": 0.6952282157676348, "pr_auc": 0.07211116528522671, "balanced_accuracy_ci95": [ 0.5, 0.5 ] }, "dolly": { "n": 527, "benign": 527, "attacks": 0, "tn": 527, "fp": 0, "fn": 0, "tp": 0, "accuracy": 1.0, "false_positive_rate": 0.0, "recall": null, "precision": null, "f1": null }, "openassistant": { "n": 1068, "benign": 1065, "attacks": 3, "tn": 1034, "fp": 31, "fn": 2, "tp": 1, "accuracy": 0.9691011235955056, "false_positive_rate": 0.02910798122065728, "recall": 0.3333333333333333, "precision": 0.03125, "f1": 0.05714285714285714, "balanced_accuracy": 0.652112676056338, "roc_auc": 0.7599374021909233, "pr_auc": 0.026601343611652888, "balanced_accuracy_ci95": [ 0.48169014084507045, 0.9859154929577465 ] }, "synthetic": { "n": 1009, "benign": 48, "attacks": 961, "tn": 33, "fp": 15, "fn": 156, "tp": 805, "accuracy": 0.8305252725470763, "false_positive_rate": 0.3125, "recall": 0.8376690946930281, "precision": 0.9817073170731707, "f1": 0.9039865244244806, "balanced_accuracy": 0.762584547346514, "roc_auc": 0.8537764481442941, "pr_auc": 0.9915786622260168, "balanced_accuracy_ci95": [ 0.6922563302115852, 0.8303007392473117 ] }, "synthetic_clean": { "n": 124, "benign": 18, "attacks": 106, "tn": 18, "fp": 0, "fn": 35, "tp": 71, "accuracy": 0.717741935483871, "false_positive_rate": 0.0, "recall": 0.6698113207547169, "precision": 1.0, "f1": 0.8022598870056498, "balanced_accuracy": 0.8349056603773585, "roc_auc": 0.9963312368972747, "pr_auc": 0.9993791997165354, "balanced_accuracy_ci95": [ 0.7877358490566038, 0.8773584905660378 ] } }, "notinject_by_category": { "Common Queries": { "n": 126, "benign": 126, "attacks": 0, "tn": 91, "fp": 35, "fn": 0, "tp": 0, "accuracy": 0.7222222222222222, "false_positive_rate": 0.2777777777777778, "recall": null, "precision": null, "f1": null }, "Multilingual": { "n": 84, "benign": 84, "attacks": 0, "tn": 35, "fp": 49, "fn": 0, "tp": 0, "accuracy": 0.4166666666666667, "false_positive_rate": 0.5833333333333334, "recall": null, "precision": null, "f1": null }, "Technique Queries": { "n": 87, "benign": 87, "attacks": 0, "tn": 37, "fp": 50, "fn": 0, "tp": 0, "accuracy": 0.42528735632183906, "false_positive_rate": 0.5747126436781609, "recall": null, "precision": null, "f1": null }, "Virtual Creation": { "n": 42, "benign": 42, "attacks": 0, "tn": 31, "fp": 11, "fn": 0, "tp": 0, "accuracy": 0.7380952380952381, "false_positive_rate": 0.2619047619047619, "recall": null, "precision": null, "f1": null } } }, { "id": "deepset/deberta-v3-base-injection", "name": "deepset DeBERTa", "revision": "80dda00d0b0d9a03917a7685e2ddbcd28e04dbb1", "total_parameters": 184423682, "latency_p50_ms": 16.56841650037677, "latency_p95_ms": 19.237654145035773, "suites": { "safeguard": { "n": 5140, "benign": 4068, "attacks": 1072, "tn": 2022, "fp": 2046, "fn": 5, "tp": 1067, "accuracy": 0.6009727626459144, "false_positive_rate": 0.5029498525073747, "recall": 0.9953358208955224, "precision": 0.34275618374558303, "f1": 0.5099163679808841, "balanced_accuracy": 0.7461929841940739, "roc_auc": 0.9219004534847884, "pr_auc": 0.7049010342693657, "balanced_accuracy_ci95": [ 0.7377841503214019, 0.7543056289349712 ] }, "notinject": { "n": 339, "benign": 339, "attacks": 0, "tn": 100, "fp": 239, "fn": 0, "tp": 0, "accuracy": 0.2949852507374631, "false_positive_rate": 0.7050147492625368, "recall": null, "precision": null, "f1": null }, "bipia": { "n": 575, "benign": 140, "attacks": 435, "tn": 0, "fp": 140, "fn": 0, "tp": 435, "accuracy": 0.7565217391304347, "false_positive_rate": 1.0, "recall": 1.0, "precision": 0.7565217391304347, "f1": 0.8613861386138614, "balanced_accuracy": 0.5, "roc_auc": 0.5859934318555009, "pr_auc": 0.8254676938106184, "balanced_accuracy_ci95": [ 0.5, 0.5 ] } }, "bipia_by_task": { "email": { "n": 181, "benign": 44, "attacks": 137, "tn": 0, "fp": 44, "fn": 0, "tp": 137, "accuracy": 0.7569060773480663, "false_positive_rate": 1.0, "recall": 1.0, "precision": 0.7569060773480663, "f1": 0.8616352201257862, "balanced_accuracy": 0.5, "roc_auc": 0.7644326476443265, "pr_auc": 0.9100127744949863, "balanced_accuracy_ci95": [ 0.5, 0.5 ] }, "table": { "n": 196, "benign": 46, "attacks": 150, "tn": 0, "fp": 46, "fn": 0, "tp": 150, "accuracy": 0.7653061224489796, "false_positive_rate": 1.0, "recall": 1.0, "precision": 0.7653061224489796, "f1": 0.8670520231213873, "balanced_accuracy": 0.5, "roc_auc": 0.7315942028985507, "pr_auc": 0.8763132605418175, "balanced_accuracy_ci95": [ 0.5, 0.5 ] }, "code": { "n": 198, "benign": 50, "attacks": 148, "tn": 0, "fp": 50, "fn": 0, "tp": 148, "accuracy": 0.7474747474747475, "false_positive_rate": 1.0, "recall": 1.0, "precision": 0.7474747474747475, "f1": 0.8554913294797688, "balanced_accuracy": 0.5, "roc_auc": 0.6263513513513513, "pr_auc": 0.7911275941365093, "balanced_accuracy_ci95": [ 0.5, 0.5 ] } }, "bipia_by_position": { "start": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 0, "tp": 150, "accuracy": 1.0, "false_positive_rate": null, "recall": 1.0, "precision": 1.0, "f1": 1.0 }, "middle": { "n": 135, "benign": 0, "attacks": 135, "tn": 0, "fp": 0, "fn": 0, "tp": 135, "accuracy": 1.0, "false_positive_rate": null, "recall": 1.0, "precision": 1.0, "f1": 1.0 }, "end": { "n": 150, "benign": 0, "attacks": 150, "tn": 0, "fp": 0, "fn": 0, "tp": 150, "accuracy": 1.0, "false_positive_rate": null, "recall": 1.0, "precision": 1.0, "f1": 1.0 } }, "safeguard_by_source": { "alpaca": { "n": 2412, "benign": 2410, "attacks": 2, "tn": 975, "fp": 1435, "fn": 0, "tp": 2, "accuracy": 0.4050580431177446, "false_positive_rate": 0.5954356846473029, "recall": 1.0, "precision": 0.0013917884481558804, "f1": 0.002779708130646282, "balanced_accuracy": 0.7022821576763485, "roc_auc": 0.8703319502074689, "pr_auc": 0.004757557313856766, "balanced_accuracy_ci95": [ 0.6923184647302905, 0.7120331950207469 ] }, "dolly": { "n": 527, "benign": 527, "attacks": 0, "tn": 395, "fp": 132, "fn": 0, "tp": 0, "accuracy": 0.7495256166982922, "false_positive_rate": 0.2504743833017078, "recall": null, "precision": null, "f1": null }, "openassistant": { "n": 1068, "benign": 1065, "attacks": 3, "tn": 640, "fp": 425, "fn": 0, "tp": 3, "accuracy": 0.6020599250936329, "false_positive_rate": 0.39906103286384975, "recall": 1.0, "precision": 0.007009345794392523, "f1": 0.013921113689095127, "balanced_accuracy": 0.8004694835680751, "roc_auc": 0.8165884194053209, "pr_auc": 0.016830056511017974, "balanced_accuracy_ci95": [ 0.7859154929577465, 0.8150352112676055 ] }, "synthetic": { "n": 1009, "benign": 48, "attacks": 961, "tn": 0, "fp": 48, "fn": 1, "tp": 960, "accuracy": 0.9514370664023786, "false_positive_rate": 1.0, "recall": 0.9989594172736732, "precision": 0.9523809523809523, "f1": 0.9751142712036567, "balanced_accuracy": 0.4994797086368366, "roc_auc": 0.5284642733263961, "pr_auc": 0.9488803129387817, "balanced_accuracy_ci95": [ 0.49791883454734653, 0.5 ] }, "synthetic_clean": { "n": 124, "benign": 18, "attacks": 106, "tn": 12, "fp": 6, "fn": 4, "tp": 102, "accuracy": 0.9193548387096774, "false_positive_rate": 0.3333333333333333, "recall": 0.9622641509433962, "precision": 0.9444444444444444, "f1": 0.9532710280373832, "balanced_accuracy": 0.8144654088050314, "roc_auc": 0.9433962264150944, "pr_auc": 0.9899277187057982, "balanced_accuracy_ci95": [ 0.7033542976939203, 0.9166666666666667 ] } }, "notinject_by_category": { "Common Queries": { "n": 126, "benign": 126, "attacks": 0, "tn": 54, "fp": 72, "fn": 0, "tp": 0, "accuracy": 0.42857142857142855, "false_positive_rate": 0.5714285714285714, "recall": null, "precision": null, "f1": null }, "Multilingual": { "n": 84, "benign": 84, "attacks": 0, "tn": 4, "fp": 80, "fn": 0, "tp": 0, "accuracy": 0.047619047619047616, "false_positive_rate": 0.9523809523809523, "recall": null, "precision": null, "f1": null }, "Technique Queries": { "n": 87, "benign": 87, "attacks": 0, "tn": 37, "fp": 50, "fn": 0, "tp": 0, "accuracy": 0.42528735632183906, "false_positive_rate": 0.5747126436781609, "recall": null, "precision": null, "f1": null }, "Virtual Creation": { "n": 42, "benign": 42, "attacks": 0, "tn": 5, "fp": 37, "fn": 0, "tp": 0, "accuracy": 0.11904761904761904, "false_positive_rate": 0.8809523809523809, "recall": null, "precision": null, "f1": null } } } ], "dataset": { "seed": 20260911, "available_codeintegrity_unique_hashes": 480637, "evaluation_sha256": "803be702553db51709dc0bbdfbaaca3b641c26f1092db0607d2e9e52a0cb0edb", "normalization": "NFKC, casefold, strip zero-width characters, collapse whitespace", "counts": { "safeguard": 5140, "notinject": 339, "bipia": 575 }, "labels": { "safeguard": { "0": 4068, "1": 1072 }, "notinject": { "0": 339 }, "bipia": { "0": 140, "1": 435 } }, "exclusions": { "safeguard:known_codeintegrity_corpus_overlap": 285, "bipia:within_suite_duplicate": 25 } } }