{ "detectors": [ "scamllm", "pimref", "t5phishing", "xgboost", "securenet_llama", "email_phishing_detection_v3", "phishing_email_agent", "rspamd", "spamassassin", "spamscanner" ], "metrics": { "email_phishing_detection_v3": { "S1": { "HW": { "accuracy": 0.8942632170978627, "f1": 0.7796875, "fnr": 0.2541106128550074, "fpr": 0.05605605605605601, "mcc": 0.7115784959654128, "n_rows": 2667, "precision": 0.8166939443535188, "recall": 0.7458893871449925, "tnr": 0.943943943943944 }, "LLM": { "accuracy": 0.619140625, "f1": 0.5416078984485191, "fnr": 0.11926605504587148, "fpr": 0.4706190975865687, "mcc": 0.3618592420331907, "n_rows": 2560, "precision": 0.3910386965376782, "recall": 0.8807339449541285, "tnr": 0.5293809024134313 } }, "S2": { "HW": { "accuracy": 0.8651629072681705, "f1": 0.8514632799558255, "fnr": 0.226680040120361, "fpr": 0.0430861723446893, "mcc": 0.7429124255872722, "n_rows": 1995, "precision": 0.9471744471744472, "recall": 0.7733199598796389, "tnr": 0.9569138276553107 }, "LLM": { "accuracy": 0.521594684385382, "f1": 0.6040329972502292, "fnr": 0.2718232044198895, "fpr": 0.6859045504994451, "mcc": 0.0464428341296914, "n_rows": 1806, "precision": 0.5160532498042286, "recall": 0.7281767955801105, "tnr": 0.3140954495005549 } }, "S4": { "HW": { "accuracy": 0.7929806714140386, "f1": 0.8209414870215574, "fnr": 0.2931818181818181, "fpr": 0.0309597523219814, "mcc": 0.63519604658193, "n_rows": 1966, "precision": 0.9790136411332634, "recall": 0.7068181818181818, "tnr": 0.9690402476780186 }, "LLM": { "accuracy": 0.7128450106157113, "f1": 0.727730246602919, "fnr": 0.4266455194290245, "fpr": 0.0048154093097913, "mcc": 0.5495905951882996, "n_rows": 1884, "precision": 0.9958677685950412, "recall": 0.5733544805709754, "tnr": 0.9951845906902087 } }, "S5": { "HW": { "accuracy": 0.8553971486761711, "f1": 0.8388561053109397, "fnr": 0.24694376528117357, "fpr": 0.0423452768729641, "mcc": 0.7261151203616962, "n_rows": 2455, "precision": 0.9467213114754098, "recall": 0.7530562347188264, "tnr": 0.9576547231270359 }, "LLM": { "accuracy": 0.9350704826997008, "f1": 0.9355932203389833, "fnr": 0.0547945205479452, "fpr": 0.0750213128729752, "mcc": 0.8703290149434587, "n_rows": 2341, "precision": 0.9261744966442953, "recall": 0.9452054794520548, "tnr": 0.9249786871270248 } }, "S6": { "HW": { "accuracy": 0.7569542925711792, "f1": 0.7198566054271676, "fnr": 0.42948611803260145, "fpr": 0.015952184820222495, "mcc": 0.5884961515160549, "n_rows": 9167, "precision": 0.9777422158781651, "recall": 0.5705138819673985, "tnr": 0.9840478151797775 }, "LLM": { "accuracy": 0.7787583056478405, "f1": 0.7692765112234913, "fnr": 0.3271816029331193, "fpr": 0.06723121215353253, "mcc": 0.6144839744398709, "n_rows": 9632, "precision": 0.9292226649298013, "recall": 0.6728183970668807, "tnr": 0.9327687878464674 } }, "S8": { "HW": { "accuracy": 0.7702228900576008, "f1": 0.7041471449559986, "fnr": 0.44966127692734553, "fpr": 0.011716827155058035, "mcc": 0.5997858726675449, "n_rows": 7986, "precision": 0.9787665788542563, "recall": 0.5503387230726545, "tnr": 0.9882831728449419 }, "LLM": { "accuracy": 0.6502126240306846, "f1": 0.43761285502610553, "fnr": 0.6866032347664177, "fpr": 0.013491106446735473, "mcc": 0.37475894645223096, "n_rows": 11993, "precision": 0.969254927186788, "recall": 0.3133967652335821, "tnr": 0.9865088935532645 } }, "overview": { "HW": { "accuracy": 0.7950907150480256, "f1": 0.7527821208498114, "fnr": 0.3809559824534866, "fpr": 0.0260488704472106, "mcc": 0.6330668819245869, "n_rows": 26236, "precision": 0.9602299389957766, "recall": 0.6190440175465134, "tnr": 0.9739511295527894 }, "LLM": { "accuracy": 0.7068440561292031, "f1": 0.6562669771051611, "fnr": 0.4485097502119611, "fpr": 0.1331048847678559, "mcc": 0.43989121253323, "n_rows": 30216, "precision": 0.8101945003353455, "recall": 0.5514902497880388, "tnr": 0.8668951152321441 } } }, "phishing_email_agent": { "S1": { "HW": { "accuracy": 0.7740366629255518, "f1": 0.3505376344086021, "fnr": 0.7578008915304606, "fpr": 0.047, "mcc": 0.2873990532126625, "n_rows": 2673, "precision": 0.6342412451361867, "recall": 0.2421991084695393, "tnr": 0.953 }, "LLM": { "accuracy": 0.6423494201271979, "f1": 0.5243781094527363, "fnr": 0.21693907875185728, "fpr": 0.405, "mcc": 0.3281824746846828, "n_rows": 2673, "precision": 0.394166043380703, "recall": 0.7830609212481426, "tnr": 0.595 } }, "S2": { "HW": { "accuracy": 0.5804511278195489, "f1": 0.3414634146341463, "fnr": 0.7823470411233701, "fpr": 0.0571142284569138, "mcc": 0.2331991469200534, "n_rows": 1995, "precision": 0.791970802919708, "recall": 0.2176529588766299, "tnr": 0.9428857715430862 }, "LLM": { "accuracy": 0.4977443609022556, "f1": 0.4085005903187721, "fnr": 0.6529588766298897, "fpr": 0.3517034068136273, "mcc": -0.0048894249849607, "n_rows": 1995, "precision": 0.4964131994261119, "recall": 0.3470411233701103, "tnr": 0.6482965931863727 } }, "S4": { "HW": { "accuracy": null, "f1": null, "fnr": null, "fpr": null, "mcc": null, "n_rows": 0, "precision": null, "recall": null, "tnr": null }, "LLM": { "accuracy": 0.614, "f1": 0.5995850622406639, "fnr": 0.5673652694610778, "fpr": 0.0210843373493975, "mcc": 0.4245688319040731, "n_rows": 2000, "precision": 0.9763513513513512, "recall": 0.4326347305389221, "tnr": 0.9789156626506025 } }, "S5": { "HW": { "accuracy": 0.5754310344827587, "f1": 0.33558178752107926, "fnr": 0.7857911733046287, "fpr": 0.0625674217907227, "mcc": 0.2195203214059418, "n_rows": 1856, "precision": 0.77431906614786, "recall": 0.2142088266953713, "tnr": 0.9374325782092773 }, "LLM": { "accuracy": 0.9169381107491856, "f1": 0.9100529100529101, "fnr": 0.1596091205211726, "fpr": 0.006514657980456, "mcc": 0.8438235859714008, "n_rows": 2456, "precision": 0.9923076923076924, "recall": 0.8403908794788274, "tnr": 0.993485342019544 } }, "S6": { "HW": { "accuracy": 0.5290721649484537, "f1": 0.3138477161740359, "fnr": 0.805, "fpr": 0.05474226804123708, "mcc": 0.2018604188770297, "n_rows": 9700, "precision": 0.8114645380671079, "recall": 0.195, "tnr": 0.945257731958763 }, "LLM": { "accuracy": 0.5398969072164949, "f1": 0.3808747886449652, "fnr": 0.7440721649484536, "fpr": 0.09628865979381442, "mcc": 0.20249341557435677, "n_rows": 9700, "precision": 0.766090015350251, "recall": 0.25592783505154637, "tnr": 0.9037113402061856 } }, "S8": { "HW": { "accuracy": 0.5674444444444444, "f1": 0.31462509434304403, "fnr": 0.8013333333333333, "fpr": 0.06377777777777777, "mcc": 0.1995470974144848, "n_rows": 9000, "precision": 0.7564210171178446, "recall": 0.19866666666666663, "tnr": 0.9362222222222223 }, "LLM": { "accuracy": 0.6311720698254364, "f1": 0.47180746210913954, "fnr": 0.6615223816999172, "fpr": 0.07626713551389094, "mcc": 0.32735383674417284, "n_rows": 12030, "precision": 0.8291713722066267, "recall": 0.3384776183000828, "tnr": 0.923732864486109 } }, "overview": { "HW": { "accuracy": 0.5639969037428285, "f1": 0.3195594244093089, "fnr": 0.7992321085762756, "fpr": 0.0580255516840882, "mcc": 0.2116163934320854, "n_rows": 43924, "precision": 0.7826313957535677, "recall": 0.20076789142372423, "tnr": 0.9419744483159118 }, "LLM": { "accuracy": 0.5984404978599892, "f1": 0.4705348165926118, "fnr": 0.6560618273197099, "fpr": 0.1271788580483566, "mcc": 0.2537210309897941, "n_rows": 40654, "precision": 0.7446109628413057, "recall": 0.3439381726802901, "tnr": 0.8728211419516434 } } }, "pimref": { "S1": { "HW": { "accuracy": 0.7463524130190797, "f1": 0.0686813186813186, "fnr": 0.962852897473997, "fpr": 0.015, "mcc": 0.0677131909571562, "n_rows": 2673, "precision": 0.4545454545454545, "recall": 0.0371471025260029, "tnr": 0.985 }, "LLM": { "accuracy": 0.7478488589599701, "f1": 0.0088235294117647, "fnr": 0.9955423476968795, "fpr": 0.002, "mcc": 0.0208719909791423, "n_rows": 2673, "precision": 0.42857142857142855, "recall": 0.0044576523031203, "tnr": 0.998 } }, "S2": { "HW": { "accuracy": 0.519298245614035, "f1": 0.0840496657115568, "fnr": 0.9558676028084252, "fpr": 0.0060120240480961, "mcc": 0.1219341030398597, "n_rows": 1995, "precision": 0.88, "recall": 0.0441323971915747, "tnr": 0.9939879759519039 }, "LLM": { "accuracy": 0.5112781954887218, "f1": 0.0469208211143695, "fnr": 0.97592778335005, "fpr": 0.002004008016032, "mcc": 0.0972904918335715, "n_rows": 1995, "precision": 0.9230769230769232, "recall": 0.0240722166499498, "tnr": 0.997995991983968 } }, "S4": { "HW": { "accuracy": 0.3422227375500029, "f1": 0.0772608978529603, "fnr": 0.9595503704334496, "fpr": 0.0141663639665819, "mcc": 0.0695533694811705, "n_rows": 17249, "precision": 0.8589511754068716, "recall": 0.0404496295665502, "tnr": 0.9858336360334181 }, "LLM": { "accuracy": 0.3211780393066265, "f1": 0.0057739662053154, "fnr": 0.9971046580941837, "fpr": 0.0, "mcc": 0.0304308804641257, "n_rows": 17249, "precision": 1.0, "recall": 0.0028953419058162, "tnr": 1.0 } }, "S5": { "HW": { "accuracy": 0.5138436482084691, "f1": 0.0815384615384615, "fnr": 0.9568403908794788, "fpr": 0.015472312703582998, "mcc": 0.0820652965681614, "n_rows": 2456, "precision": 0.7361111111111112, "recall": 0.0431596091205211, "tnr": 0.984527687296417 }, "LLM": { "accuracy": 0.500814332247557, "f1": 0.0032520325203252, "fnr": 0.998371335504886, "fpr": 0.0, "mcc": 0.0285481334769959, "n_rows": 2456, "precision": 1.0, "recall": 0.001628664495114, "tnr": 1.0 } }, "S6": { "HW": { "accuracy": 0.4608247422680412, "f1": 0.08010205310320638, "fnr": 0.9577835051546392, "fpr": 0.014536082474226771, "mcc": 0.080867234233904, "n_rows": 9700, "precision": 0.7892639698964734, "recall": 0.04221649484536082, "tnr": 0.9854639175257732 }, "LLM": { "accuracy": 0.4605154639175258, "f1": 0.06015217631113404, "fnr": 0.9686082474226804, "fpr": 0.004432989690721644, "mcc": 0.0971907011318766, "n_rows": 9700, "precision": 0.9504023133014835, "recall": 0.031391752577319555, "tnr": 0.9955670103092784 } }, "S8": { "HW": { "accuracy": 0.5137777777777778, "f1": 0.07405284440540501, "fnr": 0.9611111111111111, "fpr": 0.0113333333333333, "mcc": 0.08819129965288443, "n_rows": 9000, "precision": 0.7753911971439358, "recall": 0.03888888888888886, "tnr": 0.9886666666666667 }, "LLM": { "accuracy": 0.5188694929343308, "f1": 0.0966955157699204, "fnr": 0.9461134364948495, "fpr": 0.016441092304699242, "mcc": 0.10832606931139288, "n_rows": 12030, "precision": 0.8639351046838606, "recall": 0.053886563505150524, "tnr": 0.9835589076953007 } }, "overview": { "HW": { "accuracy": 0.4594887734123322, "f1": 0.0777283650526199, "fnr": 0.9591475276559915, "fpr": 0.0129884384604127, "mcc": 0.0831379263621951, "n_rows": 61773, "precision": 0.7985244040862656, "recall": 0.0408524723440085, "tnr": 0.9870115615395872 }, "LLM": { "accuracy": 0.4600563147730456, "f1": 0.0554466680420878, "fnr": 0.9713080844476472, "fpr": 0.0077059869590989, "mcc": 0.0758555644070627, "n_rows": 67833, "precision": 0.8212375859434683, "recall": 0.0286919155523527, "tnr": 0.9922940130409011 } } }, "rspamd": { "S1": { "HW": { "accuracy": 0.7687991021324355, "f1": 0.2444987775061124, "fnr": 0.8514115898959881, "fpr": 0.0225, "mcc": 0.24161570646244712, "n_rows": 2673, "precision": 0.6896551724137931, "recall": 0.1485884101040119, "tnr": 0.9775 }, "LLM": { "accuracy": 0.6187803965581743, "f1": 0.5218207414359456, "fnr": 0.1738484398216939, "fpr": 0.45100000000000007, "mcc": 0.3270110182281254, "n_rows": 2673, "precision": 0.3813443072702332, "recall": 0.826151560178306, "tnr": 0.5489999999999999 } }, "S2": { "HW": { "accuracy": 0.562406015037594, "f1": 0.2544833475661827, "fnr": 0.8505516549648947, "fpr": 0.0250501002004008, "mcc": 0.2204436055043101, "n_rows": 1995, "precision": 0.8563218390804598, "recall": 0.1494483450351053, "tnr": 0.9749498997995992 }, "LLM": { "accuracy": 0.4872180451127819, "f1": 0.16078753076292038, "fnr": 0.901705115346038, "fpr": 0.1242484969939879, "mcc": -0.0412648060507166, "n_rows": 1995, "precision": 0.4414414414414414, "recall": 0.0982948846539618, "tnr": 0.8757515030060121 } }, "S4": { "HW": { "accuracy": 0.426, "f1": 0.2631578947368421, "fnr": 0.8474702380952381, "fpr": 0.0137195121951219, "mcc": 0.2108266342880738, "n_rows": 2000, "precision": 0.9579439252336448, "recall": 0.1525297619047619, "tnr": 0.9862804878048781 }, "LLM": { "accuracy": 0.6065, "f1": 0.5855713533438652, "fnr": 0.5838323353293413, "fpr": 0.0105421686746987, "mcc": 0.4247466208626039, "n_rows": 2000, "precision": 0.9875666074600356, "recall": 0.4161676646706587, "tnr": 0.9894578313253013 } }, "S5": { "HW": { "accuracy": 0.5565960912052117, "f1": 0.2432244614315497, "fnr": 0.8574918566775245, "fpr": 0.0293159609120521, "mcc": 0.2019600176302813, "n_rows": 2456, "precision": 0.8293838862559242, "recall": 0.1425081433224755, "tnr": 0.9706840390879479 }, "LLM": { "accuracy": 0.8045602605863192, "f1": 0.757085020242915, "fnr": 0.3908794788273615, "fpr": 0.0, "mcc": 0.6617696805311177, "n_rows": 2456, "precision": 1.0, "recall": 0.6091205211726385, "tnr": 1.0 } }, "S6": { "HW": { "accuracy": 0.5118556701030927, "f1": 0.24392292216532913, "fnr": 0.8584020618556701, "fpr": 0.02422680412371131, "mcc": 0.20305154412113274, "n_rows": 9700, "precision": 0.8794922772982963, "recall": 0.1415979381443299, "tnr": 0.9757731958762887 }, "LLM": { "accuracy": 0.5138144329896908, "f1": 0.28221753072470895, "fnr": 0.8201030927835051, "fpr": 0.053711340206185565, "mcc": 0.18059319333210583, "n_rows": 9700, "precision": 0.7842103478361396, "recall": 0.17989690721649484, "tnr": 0.9462886597938145 } }, "S8": { "HW": { "accuracy": 0.5593333333333333, "f1": 0.2433241931687077, "fnr": 0.8582222222222222, "fpr": 0.023111111111111103, "mcc": 0.21546551895516014, "n_rows": 9000, "precision": 0.8590496620257871, "recall": 0.14177777777777775, "tnr": 0.9768888888888889 }, "LLM": { "accuracy": 0.583624272651704, "f1": 0.334119425352043, "fnr": 0.7639235245220283, "fpr": 0.06916043225270155, "mcc": 0.20734816028530217, "n_rows": 12030, "precision": 0.5797430436919332, "recall": 0.23607647547797173, "tnr": 0.9308395677472985 } }, "overview": { "HW": { "accuracy": 0.545825810334451, "f1": 0.2455726935161382, "fnr": 0.856958655685883, "fpr": 0.023440975002223998, "mcc": 0.21402688406251844, "n_rows": 46524, "precision": 0.8671205244578921, "recall": 0.1430413443141169, "tnr": 0.976559024997776 }, "LLM": { "accuracy": 0.5608124368291861, "f1": 0.3645100633748868, "fnr": 0.7570891539425442, "fpr": 0.096816114359974, "mcc": 0.193177793948862, "n_rows": 41554, "precision": 0.729884256031237, "recall": 0.2429108460574558, "tnr": 0.903183885640026 } } }, "scamllm": { "S1": { "HW": { "accuracy": 0.7751589973812196, "f1": 0.6579396698918611, "fnr": 0.1411589895988113, "fpr": 0.253, "mcc": 0.5355556536640032, "n_rows": 2673, "precision": 0.533210332103321, "recall": 0.8588410104011887, "tnr": 0.747 }, "LLM": { "accuracy": 0.6124205013093902, "f1": 0.5468066491688539, "fnr": 0.0713224368499257, "fpr": 0.494, "mcc": 0.38567335632638794, "n_rows": 2673, "precision": 0.3874767513949163, "recall": 0.9286775631500744, "tnr": 0.506 } }, "S2": { "HW": { "accuracy": 0.8, "f1": 0.812235294117647, "fnr": 0.1344032096288866, "fpr": 0.2655310621242485, "mcc": 0.6052678062288607, "n_rows": 1995, "precision": 0.7650709219858156, "recall": 0.8655967903711134, "tnr": 0.7344689378757515 }, "LLM": { "accuracy": 0.5659147869674186, "f1": 0.668199233716475, "fnr": 0.1253761283851554, "fpr": 0.7424849699398798, "mcc": 0.1679170041654918, "n_rows": 1995, "precision": 0.5406075635461872, "recall": 0.8746238716148446, "tnr": 0.2575150300601202 } }, "S4": { "HW": { "accuracy": 0.8341932865673373, "f1": 0.8779030054644809, "fnr": 0.12441454483522091, "fpr": 0.2540864511442063, "mcc": 0.6196671099166682, "n_rows": 17249, "precision": 0.880232856775961, "recall": 0.875585455164779, "tnr": 0.7459135488557938 }, "LLM": { "accuracy": 0.6214853034958548, "f1": 0.6329754342571252, "fnr": 0.520565443242783, "fpr": 0.0755539411551035, "mcc": 0.394602110859765, "n_rows": 17249, "precision": 0.931194177968905, "recall": 0.4794345567572171, "tnr": 0.9244460588448965 } }, "S5": { "HW": { "accuracy": 0.7968241042345277, "f1": 0.8114846996599925, "fnr": 0.1254071661237785, "fpr": 0.2809446254071661, "mcc": 0.6009619180968087, "n_rows": 2456, "precision": 0.7568710359408034, "recall": 0.8745928338762216, "tnr": 0.7190553745928339 }, "LLM": { "accuracy": 0.9275244299674267, "f1": 0.9298107255520502, "fnr": 0.0399022801302931, "fpr": 0.1050488599348534, "mcc": 0.8568691024656853, "n_rows": 2456, "precision": 0.9013761467889908, "recall": 0.9600977198697068, "tnr": 0.8949511400651466 } }, "S6": { "HW": { "accuracy": 0.8217525773195876, "f1": 0.842946153159894, "fnr": 0.1227319587628866, "fpr": 0.24670103092783505, "mcc": 0.6334877546509882, "n_rows": 9700, "precision": 0.8126519540979276, "recall": 0.8772680412371134, "tnr": 0.753298969072165 }, "LLM": { "accuracy": 0.7394845360824742, "f1": 0.7873675871136583, "fnr": 0.16412371134020617, "fpr": 0.41711340206185565, "mcc": 0.42202719579491227, "n_rows": 9700, "precision": 0.744442319223504, "recall": 0.8358762886597938, "tnr": 0.5828865979381443 } }, "S8": { "HW": { "accuracy": 0.814, "f1": 0.8244901902402829, "fnr": 0.1262222222222222, "fpr": 0.24577777777777773, "mcc": 0.6325508093287692, "n_rows": 9000, "precision": 0.7804784222881386, "recall": 0.8737777777777778, "tnr": 0.7542222222222222 }, "LLM": { "accuracy": 0.7400665004156276, "f1": 0.7169971004144635, "fnr": 0.31962926762406957, "fpr": 0.20025972946421822, "mcc": 0.48982184951379903, "n_rows": 12030, "precision": 0.775976223919984, "recall": 0.6803707323759305, "tnr": 0.7997402705357818 } }, "overview": { "HW": { "accuracy": 0.8192576044550208, "f1": 0.8436953143593118, "fnr": 0.1250834760895444, "fpr": 0.2508780916142251, "mcc": 0.6323620744270725, "n_rows": 61773, "precision": 0.814625574479589, "recall": 0.8749165239104556, "tnr": 0.749121908385775 }, "LLM": { "accuracy": 0.7063818495422582, "f1": 0.7187063060518325, "fnr": 0.3208957215683135, "fpr": 0.2599617993808865, "mcc": 0.4169001832048174, "n_rows": 67833, "precision": 0.7632131501589777, "recall": 0.6791042784316865, "tnr": 0.7400382006191135 } } }, "securenet_llama": { "S1": { "HW": { "accuracy": 0.9180695847362514, "f1": 0.8332063975628332, "fnr": 0.1872213967310549, "fpr": 0.0465, "mcc": 0.7793823250845866, "n_rows": 2673, "precision": 0.8546875, "recall": 0.812778603268945, "tnr": 0.9535 }, "LLM": { "accuracy": 0.6210250654695099, "f1": 0.5531539479488311, "fnr": 0.0683506686478454, "fpr": 0.48349999999999993, "mcc": 0.39645180716401984, "n_rows": 2673, "precision": 0.3933500627352572, "recall": 0.9316493313521546, "tnr": 0.5165000000000001 } }, "S2": { "HW": { "accuracy": 0.8987468671679198, "f1": 0.8926673751328374, "fnr": 0.1574724172517552, "fpr": 0.0450901803607214, "mcc": 0.8025578123763634, "n_rows": 1995, "precision": 0.9491525423728814, "recall": 0.8425275827482447, "tnr": 0.9549098196392786 }, "LLM": { "accuracy": 0.6015037593984962, "f1": 0.6964490263459335, "fnr": 0.0852557673019057, "fpr": 0.7114228456913828, "mcc": 0.2607450500846078, "n_rows": 1995, "precision": 0.562268803945746, "recall": 0.9147442326980942, "tnr": 0.2885771543086172 } }, "S4": { "HW": { "accuracy": 0.871564420735243, "f1": 0.8983711860518467, "fnr": 0.166098807495741, "fpr": 0.0481293134762077, "mcc": 0.7429557573241693, "n_rows": 17246, "precision": 0.9736449527598209, "recall": 0.833901192504259, "tnr": 0.9518706865237923 }, "LLM": { "accuracy": 0.7660537154127269, "f1": 0.7927863124903664, "fnr": 0.342452910594051, "fpr": 0.0027243007628042, "mcc": 0.6138881542439424, "n_rows": 17239, "precision": 0.9980595084087968, "recall": 0.6575470894059491, "tnr": 0.9972756992371958 } }, "S5": { "HW": { "accuracy": 0.8880293159609121, "f1": 0.8810038944180009, "fnr": 0.1710097719869706, "fpr": 0.0529315960912052, "mcc": 0.7815259512249114, "n_rows": 2456, "precision": 0.9399815327793168, "recall": 0.8289902280130294, "tnr": 0.9470684039087948 }, "LLM": { "accuracy": 0.8745928338762216, "f1": 0.887836853605244, "fnr": 0.007328990228012999, "fpr": 0.2434853420195439, "mcc": 0.7709931550777308, "n_rows": 2456, "precision": 0.803030303030303, "recall": 0.992671009771987, "tnr": 0.7565146579804561 } }, "S6": { "HW": { "accuracy": 0.8857731958762887, "f1": 0.8895088953935278, "fnr": 0.16824742268041237, "fpr": 0.046185567010309264, "mcc": 0.7765777025867412, "n_rows": 9700, "precision": 0.9559817020253627, "recall": 0.8317525773195876, "tnr": 0.9538144329896907 }, "LLM": { "accuracy": 0.8239480198019802, "f1": 0.8226495383009718, "fnr": 0.2689037438887366, "fpr": 0.04361327297129335, "mcc": 0.6915986238037272, "n_rows": 9696, "precision": 0.9606997248485533, "recall": 0.7310962561112634, "tnr": 0.9563867270287066 } }, "S8": { "HW": { "accuracy": 0.8896666666666667, "f1": 0.8819307922964846, "fnr": 0.17555555555555552, "fpr": 0.0451111111111111, "mcc": 0.7860633719984291, "n_rows": 9000, "precision": 0.9480827193243961, "recall": 0.8244444444444444, "tnr": 0.9548888888888889 }, "LLM": { "accuracy": 0.6914380714879468, "f1": 0.5776556265320245, "fnr": 0.49302965946253496, "fpr": 0.12402889230823648, "mcc": 0.4346711488331501, "n_rows": 12030, "precision": 0.8829995884760697, "recall": 0.506970340537465, "tnr": 0.8759711076917636 } }, "overview": { "HW": { "accuracy": 0.8848470131131617, "f1": 0.8893899575473898, "fnr": 0.1696091526801788, "fpr": 0.0465388555539294, "mcc": 0.7790695730837268, "n_rows": 61770, "precision": 0.9574140413137366, "recall": 0.8303908473198212, "tnr": 0.9534611444460706 }, "LLM": { "accuracy": 0.7495096954950969, "f1": 0.7408029052291072, "fnr": 0.3518197110891565, "fpr": 0.1255104729284679, "mcc": 0.5276426082132066, "n_rows": 67815, "precision": 0.8643096204514705, "recall": 0.6481802889108434, "tnr": 0.874489527071532 } } }, "spamassassin": { "S1": { "HW": { "accuracy": 0.8387579498690609, "f1": 0.576204523107178, "fnr": 0.5646359583952452, "fpr": 0.025499999999999995, "mcc": 0.5312493144695061, "n_rows": 2673, "precision": 0.8517441860465116, "recall": 0.4353640416047548, "tnr": 0.9745 }, "LLM": { "accuracy": 0.6861204638982417, "f1": 0.4693232131562302, "fnr": 0.4487369985141159, "fpr": 0.2685, "mcc": 0.2591372429242004, "n_rows": 2673, "precision": 0.408590308370044, "recall": 0.5512630014858841, "tnr": 0.7315 } }, "S2": { "HW": { "accuracy": 0.7042606516290727, "f1": 0.59366391184573, "fnr": 0.567703109327984, "fpr": 0.0240480961923847, "mcc": 0.4864875181658814, "n_rows": 1995, "precision": 0.9472527472527472, "recall": 0.432296890672016, "tnr": 0.9759519038076153 }, "LLM": { "accuracy": 0.5213032581453634, "f1": 0.2254663422546634, "fnr": 0.8605817452357071, "fpr": 0.0971943887775551, "mcc": 0.0653705175050706, "n_rows": 1995, "precision": 0.5889830508474576, "recall": 0.1394182547642928, "tnr": 0.902805611222445 } }, "S4": { "HW": { "accuracy": 0.6, "f1": 0.5837669094693028, "fnr": 0.5825892857142857, "fpr": 0.0259146341463414, "mcc": 0.4054760944139384, "n_rows": 2000, "precision": 0.9705882352941176, "recall": 0.4174107142857143, "tnr": 0.9740853658536586 }, "LLM": { "accuracy": 0.4645, "f1": 0.3368421052631579, "fnr": 0.7964071856287425, "fpr": 0.0105421686746987, "mcc": 0.262401304948961, "n_rows": 2000, "precision": 0.974910394265233, "recall": 0.2035928143712574, "tnr": 0.9894578313253013 } }, "S5": { "HW": { "accuracy": null, "f1": null, "fnr": null, "fpr": null, "mcc": null, "n_rows": 0, "precision": null, "recall": null, "tnr": null }, "LLM": { "accuracy": 0.7129478827361564, "f1": 0.597372929754426, "fnr": 0.5741042345276873, "fpr": 0.0, "mcc": 0.5201575629806013, "n_rows": 2456, "precision": 1.0, "recall": 0.4258957654723127, "tnr": 1.0 } }, "S6": { "HW": { "accuracy": 0.6552577319587629, "f1": 0.567222116796478, "fnr": 0.594278350515464, "fpr": 0.030824742268041213, "mcc": 0.4342848883822958, "n_rows": 9700, "precision": 0.9433038179766366, "recall": 0.4057216494845361, "tnr": 0.9691752577319588 }, "LLM": { "accuracy": 0.6401030927835052, "f1": 0.5076237530251314, "fnr": 0.6173711340206186, "fpr": 0.07278350515463915, "mcc": 0.35880192258745025, "n_rows": 9700, "precision": 0.8875318313581659, "recall": 0.38262886597938145, "tnr": 0.9272164948453608 } }, "S8": { "HW": { "accuracy": null, "f1": null, "fnr": null, "fpr": null, "mcc": null, "n_rows": 0, "precision": null, "recall": null, "tnr": null }, "LLM": { "accuracy": 0.5873980054397099, "f1": 0.3826149622336526, "fnr": 0.7329932630376055, "fpr": 0.09243603535986783, "mcc": 0.22093347627197535, "n_rows": 11030, "precision": 0.7277632924552565, "recall": 0.26700673696239446, "tnr": 0.9075639646401321 } }, "overview": { "HW": { "accuracy": 0.673584471382538, "f1": 0.5706645138503457, "fnr": 0.5906326914724193, "fpr": 0.0285621021707197, "mcc": 0.4513849749343391, "n_rows": 26068, "precision": 0.9417152373022483, "recall": 0.4093673085275807, "tnr": 0.9714378978292803 }, "LLM": { "accuracy": 0.6181675683875208, "f1": 0.48589032236103075, "fnr": 0.6526500219010074, "fpr": 0.089072446993213, "mcc": 0.3098834583469269, "n_rows": 39554, "precision": 0.8082672706681767, "recall": 0.3473499780989925, "tnr": 0.910927553006787 } } }, "spamscanner": { "S1": { "HW": { "accuracy": 0.7622613253463122, "f1": 0.11188811188811182, "fnr": 0.940387481371088, "fpr": 0.002, "mcc": 0.19630743334795311, "n_rows": 2671, "precision": 0.9090909090909092, "recall": 0.05961251862891201, "tnr": 0.998 }, "LLM": { "accuracy": 0.7471006359895249, "f1": 0.0424929178470254, "fnr": 0.9777117384843982, "fpr": 0.009, "mcc": 0.0522314115893126, "n_rows": 2673, "precision": 0.4545454545454545, "recall": 0.0222882615156017, "tnr": 0.991 } }, "S2": { "HW": { "accuracy": 0.5241206030150753, "f1": 0.090297790585975, "fnr": 0.9526209677419356, "fpr": 0.002004008016032, "mcc": 0.1463953118311056, "n_rows": 1990, "precision": 0.9591836734693876, "recall": 0.0473790322580645, "tnr": 0.997995991983968 }, "LLM": { "accuracy": 0.4837092731829573, "f1": 0.0134099616858237, "fnr": 0.9929789368104311, "fpr": 0.0400801603206412, "mcc": -0.1089834201768739, "n_rows": 1995, "precision": 0.1489361702127659, "recall": 0.0070210631895687, "tnr": 0.9599198396793588 } }, "S4": { "HW": { "accuracy": 0.3603015075376884, "f1": 0.0887616320687186, "fnr": 0.9535580524344568, "fpr": 0.0, "mcc": 0.125609459579302, "n_rows": 1990, "precision": 1.0, "recall": 0.046441947565543, "tnr": 1.0 }, "LLM": { "accuracy": 0.3365, "f1": 0.0133828996282527, "fnr": 0.9932634730538922, "fpr": 0.0, "mcc": 0.0473987005281865, "n_rows": 2000, "precision": 1.0, "recall": 0.0067365269461077, "tnr": 1.0 } }, "S5": { "HW": { "accuracy": 0.5206206614944875, "f1": 0.081377151799687, "fnr": 0.9574119574119574, "fpr": 0.004071661237785, "mcc": 0.1277275194203058, "n_rows": 2449, "precision": 0.912280701754386, "recall": 0.0425880425880425, "tnr": 0.995928338762215 }, "LLM": { "accuracy": 0.501628664495114, "f1": 0.0064935064935064, "fnr": 0.996742671009772, "fpr": 0.0, "mcc": 0.0403896195861009, "n_rows": 2456, "precision": 1.0, "recall": 0.003257328990228, "tnr": 1.0 } }, "S6": { "HW": { "accuracy": 0.4678453267162945, "f1": 0.08136319385389074, "fnr": 0.9575174301980152, "fpr": 0.0010303641251221834, "mcc": 0.13361291727458222, "n_rows": 9672, "precision": 0.9849844828467555, "recall": 0.04248256980198466, "tnr": 0.9989696358748779 }, "LLM": { "accuracy": 0.4665291387313048, "f1": 0.07993533183595318, "fnr": 0.9576622254126367, "fpr": 0.0057778642024366045, "mcc": 0.11280909550782543, "n_rows": 9695, "precision": 0.9502874879087361, "recall": 0.04233777458736337, "tnr": 0.9942221357975634 } }, "S8": { "HW": { "accuracy": 0.5225906966392165, "f1": 0.08717908573364477, "fnr": 0.9543006461969695, "fpr": 0.0019997032420802398, "mcc": 0.14284975664545801, "n_rows": 8986, "precision": 0.9564646211910095, "recall": 0.04569935380303032, "tnr": 0.9980002967579198 }, "LLM": { "accuracy": 0.5129675810473815, "f1": 0.05233889308683613, "fnr": 0.9728969743634488, "fpr": 0.0014949621433199274, "mcc": 0.10067826504160057, "n_rows": 12030, "precision": 0.9099710090267595, "recall": 0.02710302563655112, "tnr": 0.9985050378566801 } }, "overview": { "HW": { "accuracy": 0.5065710099965529, "f1": 0.085526053104412, "fnr": 0.9552575510715628, "fpr": 0.0016459806930913, "mcc": 0.141093727974678, "n_rows": 46416, "precision": 0.9666064981949458, "recall": 0.0447424489284371, "tnr": 0.9983540193069087 }, "LLM": { "accuracy": 0.4954987483150395, "f1": 0.061607342735616695, "fnr": 0.9680609071073766, "fpr": 0.0052992051192321, "mcc": 0.0972162424676099, "n_rows": 41544, "precision": 0.8664987405541562, "recall": 0.0319390928926233, "tnr": 0.9947007948807679 } } }, "t5phishing": { "S1": { "HW": { "accuracy": 0.5604190048634493, "f1": 0.43806791009086565, "fnr": 0.3194650817236255, "fpr": 0.48, "mcc": 0.1744024401110685, "n_rows": 2673, "precision": 0.3229901269393512, "recall": 0.6805349182763745, "tnr": 0.52 }, "LLM": { "accuracy": 0.48522259633370735, "f1": 0.3773755656108597, "fnr": 0.3803863298662704, "fpr": 0.56, "mcc": 0.052341033123951, "n_rows": 2673, "precision": 0.2713077423552374, "recall": 0.6196136701337296, "tnr": 0.43999999999999995 } }, "S2": { "HW": { "accuracy": 0.5849624060150376, "f1": 0.6173752310536045, "fnr": 0.3299899699097292, "fpr": 0.5, "mcc": 0.1725189465722719, "n_rows": 1995, "precision": 0.572407883461868, "recall": 0.6700100300902708, "tnr": 0.5 }, "LLM": { "accuracy": 0.5097744360902255, "f1": 0.5501379944802208, "fnr": 0.4002006018054162, "fpr": 0.5801603206412825, "mcc": 0.0199649879163617, "n_rows": 1995, "precision": 0.508071367884452, "recall": 0.5997993981945837, "tnr": 0.41983967935871747 } }, "S4": { "HW": { "accuracy": 0.5762073163661662, "f1": 0.6728135350461015, "fnr": 0.359959124584859, "fpr": 0.5599346167816927, "mcc": 0.0767239252790509, "n_rows": 17249, "precision": 0.7091235022171903, "recall": 0.640040875415141, "tnr": 0.44006538321830735 }, "LLM": { "accuracy": 0.49614470404081384, "f1": 0.6021150940804835, "fnr": 0.4400068125691901, "fpr": 0.6400290592081366, "mcc": -0.0757371534528859, "n_rows": 17249, "precision": 0.6510891089108911, "recall": 0.5599931874308098, "tnr": 0.3599709407918634 } }, "S5": { "HW": { "accuracy": 0.5647394136807817, "f1": 0.6024544440312384, "fnr": 0.3403908794788273, "fpr": 0.5301302931596091, "mcc": 0.1318743937083747, "n_rows": 2456, "precision": 0.5544147843942505, "recall": 0.6596091205211726, "tnr": 0.46986970684039087 }, "LLM": { "accuracy": 0.4849348534201953, "f1": 0.5295648940126441, "fnr": 0.4201954397394137, "fpr": 0.6099348534201955, "mcc": -0.0306877519950305, "n_rows": 2456, "precision": 0.4873374401095141, "recall": 0.5798045602605864, "tnr": 0.3900651465798045 } }, "S6": { "HW": { "accuracy": 0.5522680412371134, "f1": 0.6100922175011981, "fnr": 0.36350515463917527, "fpr": 0.5535051546391753, "mcc": 0.08359489461463472, "n_rows": 9700, "precision": 0.5906887673905354, "recall": 0.6364948453608248, "tnr": 0.44649484536082473 }, "LLM": { "accuracy": 0.4722680412371134, "f1": 0.5374510373327205, "fnr": 0.4435051546391753, "fpr": 0.6335051546391752, "mcc": -0.0760267993461562, "n_rows": 9700, "precision": 0.5256465019712669, "recall": 0.5564948453608247, "tnr": 0.3664948453608248 } }, "S8": { "HW": { "accuracy": 0.5183333333333333, "f1": 0.5601217656012176, "fnr": 0.38666666666666666, "fpr": 0.5766666666666667, "mcc": 0.037346975804853036, "n_rows": 9000, "precision": 0.5154061624649859, "recall": 0.6133333333333333, "tnr": 0.42333333333333334 }, "LLM": { "accuracy": 0.4933499584372402, "f1": 0.538734839000527, "fnr": 0.40705104215955257, "fpr": 0.6061513678730318, "mcc": -0.011993646057372326, "n_rows": 12030, "precision": 0.49540561487511275, "recall": 0.5929489578404475, "tnr": 0.3938486321269682 } }, "overview": { "HW": { "accuracy": 0.5509688698946141, "f1": 0.6114799562988487, "fnr": 0.36622049301704357, "fpr": 0.5533806527147666, "mcc": 0.0814512404386703, "n_rows": 61773, "precision": 0.5906962898817417, "recall": 0.6337795069829564, "tnr": 0.44661934728523345 }, "LLM": { "accuracy": 0.487889375377766, "f1": 0.5522761251739959, "fnr": 0.4281634505031094, "fpr": 0.6156885990910888, "mcc": -0.0443596734932028, "n_rows": 67833, "precision": 0.5340096208967873, "recall": 0.5718365494968906, "tnr": 0.38431140090891125 } } }, "xgboost": { "S1": { "HW": { "accuracy": 0.6849981294425739, "f1": 0.5064478311840562, "fnr": 0.3580980683506686, "fpr": 0.3005, "mcc": 0.3043101367334244, "n_rows": 2673, "precision": 0.4181994191674734, "recall": 0.6419019316493314, "tnr": 0.6995 }, "LLM": { "accuracy": 0.5952113729891507, "f1": 0.5270979020979021, "fnr": 0.1040118870728083, "fpr": 0.506, "mcc": 0.3461344313598293, "n_rows": 2673, "precision": 0.373374613003096, "recall": 0.8959881129271916, "tnr": 0.494 } }, "S2": { "HW": { "accuracy": 0.668671679197995, "f1": 0.6646372399797057, "fnr": 0.3430290872617854, "fpr": 0.3196392785571142, "mcc": 0.337425243955587, "n_rows": 1995, "precision": 0.6724845995893224, "recall": 0.6569709127382146, "tnr": 0.6803607214428857 }, "LLM": { "accuracy": 0.4917293233082707, "f1": 0.4245175936435868, "fnr": 0.6248746238716149, "fpr": 0.3917835671342685, "mcc": -0.0171300020122101, "n_rows": 1995, "precision": 0.4888888888888889, "recall": 0.3751253761283851, "tnr": 0.6082164328657316 } }, "S4": { "HW": { "accuracy": 0.6632848281059771, "f1": 0.7221318534111568, "fnr": 0.3573192540236737, "fpr": 0.2927715219760261, "mcc": 0.3268622265732748, "n_rows": 17249, "precision": 0.8239982530843979, "recall": 0.6426807459763263, "tnr": 0.7072284780239739 }, "LLM": { "accuracy": 0.6606759812163024, "f1": 0.7240842879366426, "fnr": 0.3459933577450396, "fpr": 0.3250998910279695, "mcc": 0.3081367848399423, "n_rows": 17249, "precision": 0.8109820485744457, "recall": 0.6540066422549604, "tnr": 0.6749001089720306 } }, "S5": { "HW": { "accuracy": 0.6750814332247557, "f1": 0.6672226855713094, "fnr": 0.3485342019543974, "fpr": 0.3013029315960912, "mcc": 0.3505540915460546, "n_rows": 2456, "precision": 0.6837606837606838, "recall": 0.6514657980456026, "tnr": 0.6986970684039089 }, "LLM": { "accuracy": 0.8697068403908795, "f1": 0.850187265917603, "fnr": 0.260586319218241, "fpr": 0.0, "mcc": 0.7658742112733266, "n_rows": 2456, "precision": 1.0, "recall": 0.739413680781759, "tnr": 1.0 } }, "S6": { "HW": { "accuracy": 0.6743298969072165, "f1": 0.6860839980824213, "fnr": 0.3541237113402062, "fpr": 0.2907216494845361, "mcc": 0.349393815111991, "n_rows": 9700, "precision": 0.734116047928376, "recall": 0.6458762886597939, "tnr": 0.7092783505154638 }, "LLM": { "accuracy": 0.6350515463917525, "f1": 0.6377691381019174, "fnr": 0.39778350515463917, "fpr": 0.3588659793814433, "mcc": 0.24734515531992257, "n_rows": 9700, "precision": 0.691942635394605, "recall": 0.6022164948453608, "tnr": 0.6411340206185567 } }, "S8": { "HW": { "accuracy": 0.6731111111111111, "f1": 0.6653115949463778, "fnr": 0.3502222222222222, "fpr": 0.3035555555555556, "mcc": 0.3466051689323627, "n_rows": 9000, "precision": 0.6816135437532947, "recall": 0.6497777777777778, "tnr": 0.6964444444444444 }, "LLM": { "accuracy": 0.5973399833748961, "f1": 0.6376052761130561, "fnr": 0.2800266588531543, "fpr": 0.5251292198259991, "mcc": 0.2094211824719967, "n_rows": 12030, "precision": 0.5858021850503156, "recall": 0.7199733411468456, "tnr": 0.47487078017400086 } }, "overview": { "HW": { "accuracy": 0.6711993913198323, "f1": 0.6865924977240113, "fnr": 0.3540257251531605, "fpr": 0.2970144885116347, "mcc": 0.3466905018571168, "n_rows": 61773, "precision": 0.7326615293420272, "recall": 0.6459742748468396, "tnr": 0.7029855114883653 }, "LLM": { "accuracy": 0.6309023631565757, "f1": 0.6640726677489903, "fnr": 0.3394987589078388, "fpr": 0.405618125535138, "mcc": 0.2545812922274054, "n_rows": 67833, "precision": 0.6676829268292683, "recall": 0.6605012410921611, "tnr": 0.594381874464862 } } } }, "stages": [ "overview", "S1", "S2", "S4", "S5", "S6", "S8" ] }